2024-12-09 01:40:47,728 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-09 01:40:47,749 main DEBUG Took 0.018531 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 01:40:47,750 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 01:40:47,751 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 01:40:47,752 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 01:40:47,754 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:40:47,766 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 01:40:47,793 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:40:47,795 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:40:47,795 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:40:47,796 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:40:47,797 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:40:47,797 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:40:47,798 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:40:47,799 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:40:47,799 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:40:47,800 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:40:47,801 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:40:47,801 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:40:47,802 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:40:47,802 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:40:47,803 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:40:47,803 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:40:47,804 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:40:47,804 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:40:47,805 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:40:47,805 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:40:47,806 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:40:47,806 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:40:47,807 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:40:47,807 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 01:40:47,808 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:40:47,808 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 01:40:47,810 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 01:40:47,812 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 01:40:47,814 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 01:40:47,814 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 01:40:47,816 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 01:40:47,817 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 01:40:47,828 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 01:40:47,831 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 01:40:47,835 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 01:40:47,835 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 01:40:47,836 main DEBUG createAppenders(={Console}) 2024-12-09 01:40:47,836 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-12-09 01:40:47,837 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-12-09 01:40:47,837 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-12-09 01:40:47,838 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 01:40:47,838 main DEBUG OutputStream closed 2024-12-09 01:40:47,839 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 01:40:47,839 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 01:40:47,839 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-12-09 01:40:47,948 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 01:40:47,951 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 01:40:47,952 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 01:40:47,954 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 01:40:47,955 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 01:40:47,955 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 01:40:47,956 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 01:40:47,956 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 01:40:47,957 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 01:40:47,957 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 01:40:47,958 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 01:40:47,958 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 01:40:47,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 01:40:47,959 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 01:40:47,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 01:40:47,960 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 01:40:47,961 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 01:40:47,962 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 01:40:47,965 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 01:40:47,966 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-12-09 01:40:47,966 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 01:40:47,967 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-12-09T01:40:48,384 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029 2024-12-09 01:40:48,389 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 01:40:48,389 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T01:40:48,402 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-12-09T01:40:48,445 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-09T01:40:48,481 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/cluster_fb15f3cb-c4dd-1da4-630e-097361e28d71, deleteOnExit=true 2024-12-09T01:40:48,481 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-09T01:40:48,482 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/test.cache.data in system properties and HBase conf 2024-12-09T01:40:48,483 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/hadoop.tmp.dir in system properties and HBase conf 2024-12-09T01:40:48,483 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/hadoop.log.dir in system properties and HBase conf 2024-12-09T01:40:48,484 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-09T01:40:48,485 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-09T01:40:48,485 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-09T01:40:48,621 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T01:40:48,785 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T01:40:48,799 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-09T01:40:48,800 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-09T01:40:48,804 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-09T01:40:48,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T01:40:48,806 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-09T01:40:48,808 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-09T01:40:48,809 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T01:40:48,809 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T01:40:48,810 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-09T01:40:48,811 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/nfs.dump.dir in system properties and HBase conf 2024-12-09T01:40:48,811 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/java.io.tmpdir in system properties and HBase conf 2024-12-09T01:40:48,812 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T01:40:48,812 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-09T01:40:48,813 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-09T01:40:49,841 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-09T01:40:49,943 INFO [Time-limited test {}] log.Log(170): Logging initialized @3377ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-09T01:40:50,023 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T01:40:50,091 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T01:40:50,123 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T01:40:50,123 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T01:40:50,125 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T01:40:50,139 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T01:40:50,157 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/hadoop.log.dir/,AVAILABLE} 2024-12-09T01:40:50,159 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T01:40:50,398 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2341a969{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/java.io.tmpdir/jetty-localhost-38499-hadoop-hdfs-3_4_1-tests_jar-_-any-17424401221594077594/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T01:40:50,406 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:38499} 2024-12-09T01:40:50,406 INFO [Time-limited test {}] server.Server(415): Started @3841ms 2024-12-09T01:40:50,899 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T01:40:50,906 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T01:40:50,908 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T01:40:50,908 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T01:40:50,909 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-09T01:40:50,910 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6ff64ba0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/hadoop.log.dir/,AVAILABLE} 2024-12-09T01:40:50,910 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4828e07b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T01:40:51,039 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@51084f0f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/java.io.tmpdir/jetty-localhost-33729-hadoop-hdfs-3_4_1-tests_jar-_-any-16701031692807474066/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T01:40:51,040 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@370621d9{HTTP/1.1, (http/1.1)}{localhost:33729} 2024-12-09T01:40:51,040 INFO [Time-limited test {}] server.Server(415): Started @4475ms 2024-12-09T01:40:51,113 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T01:40:51,758 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/cluster_fb15f3cb-c4dd-1da4-630e-097361e28d71/data/data1/current/BP-1038248241-172.17.0.2-1733708449557/current, will proceed with Du for space computation calculation, 2024-12-09T01:40:51,758 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/cluster_fb15f3cb-c4dd-1da4-630e-097361e28d71/data/data2/current/BP-1038248241-172.17.0.2-1733708449557/current, will proceed with Du for space computation calculation, 2024-12-09T01:40:51,854 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T01:40:51,924 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x87e11e988c4d1dd5 with lease ID 0xc15f99d8f3fc69f7: Processing first storage report for DS-ef666e65-0c86-49cb-826c-7c6d3ccb7b06 from datanode DatanodeRegistration(127.0.0.1:33243, datanodeUuid=36b2a1a8-de5a-4f07-95b2-5596a0203735, infoPort=39395, infoSecurePort=0, ipcPort=36691, storageInfo=lv=-57;cid=testClusterID;nsid=348270259;c=1733708449557) 2024-12-09T01:40:51,928 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x87e11e988c4d1dd5 with lease ID 0xc15f99d8f3fc69f7: from storage DS-ef666e65-0c86-49cb-826c-7c6d3ccb7b06 node DatanodeRegistration(127.0.0.1:33243, datanodeUuid=36b2a1a8-de5a-4f07-95b2-5596a0203735, infoPort=39395, infoSecurePort=0, ipcPort=36691, storageInfo=lv=-57;cid=testClusterID;nsid=348270259;c=1733708449557), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-12-09T01:40:51,929 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x87e11e988c4d1dd5 with lease ID 0xc15f99d8f3fc69f7: Processing first storage report for DS-97883709-3222-432c-982a-c43b657df162 from datanode DatanodeRegistration(127.0.0.1:33243, datanodeUuid=36b2a1a8-de5a-4f07-95b2-5596a0203735, infoPort=39395, infoSecurePort=0, ipcPort=36691, storageInfo=lv=-57;cid=testClusterID;nsid=348270259;c=1733708449557) 2024-12-09T01:40:51,929 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x87e11e988c4d1dd5 with lease ID 0xc15f99d8f3fc69f7: from storage DS-97883709-3222-432c-982a-c43b657df162 node DatanodeRegistration(127.0.0.1:33243, datanodeUuid=36b2a1a8-de5a-4f07-95b2-5596a0203735, infoPort=39395, infoSecurePort=0, ipcPort=36691, storageInfo=lv=-57;cid=testClusterID;nsid=348270259;c=1733708449557), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-09T01:40:51,969 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029 2024-12-09T01:40:52,091 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/cluster_fb15f3cb-c4dd-1da4-630e-097361e28d71/zookeeper_0, clientPort=50027, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/cluster_fb15f3cb-c4dd-1da4-630e-097361e28d71/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/cluster_fb15f3cb-c4dd-1da4-630e-097361e28d71/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T01:40:52,106 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50027 2024-12-09T01:40:52,125 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T01:40:52,132 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T01:40:52,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741825_1001 (size=7) 2024-12-09T01:40:52,923 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb with version=8 2024-12-09T01:40:52,923 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/hbase-staging 2024-12-09T01:40:53,019 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T01:40:53,283 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ef6f18c58dc9:0 server-side Connection retries=45 2024-12-09T01:40:53,296 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T01:40:53,297 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T01:40:53,303 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T01:40:53,303 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T01:40:53,303 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T01:40:53,487 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-09T01:40:53,557 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T01:40:53,570 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T01:40:53,575 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T01:40:53,614 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1565 (auto-detected) 2024-12-09T01:40:53,616 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-09T01:40:53,642 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37995 2024-12-09T01:40:53,673 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37995 connecting to ZooKeeper ensemble=127.0.0.1:50027 2024-12-09T01:40:53,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:379950x0, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T01:40:53,718 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37995-0x100746f49bc0000 connected 2024-12-09T01:40:53,763 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T01:40:53,766 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T01:40:53,777 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T01:40:53,782 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb, hbase.cluster.distributed=false 2024-12-09T01:40:53,815 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T01:40:53,823 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37995 2024-12-09T01:40:53,823 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37995 2024-12-09T01:40:53,824 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37995 2024-12-09T01:40:53,838 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37995 2024-12-09T01:40:53,838 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37995 2024-12-09T01:40:53,999 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ef6f18c58dc9:0 server-side Connection retries=45 2024-12-09T01:40:54,001 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T01:40:54,002 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T01:40:54,003 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T01:40:54,003 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T01:40:54,003 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T01:40:54,006 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T01:40:54,009 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T01:40:54,011 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38627 2024-12-09T01:40:54,014 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38627 connecting to ZooKeeper ensemble=127.0.0.1:50027 2024-12-09T01:40:54,016 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T01:40:54,019 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T01:40:54,026 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:386270x0, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T01:40:54,027 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:386270x0, quorum=127.0.0.1:50027, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T01:40:54,028 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38627-0x100746f49bc0001 connected 2024-12-09T01:40:54,032 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T01:40:54,042 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T01:40:54,045 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38627-0x100746f49bc0001, quorum=127.0.0.1:50027, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T01:40:54,051 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38627-0x100746f49bc0001, quorum=127.0.0.1:50027, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T01:40:54,052 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38627 2024-12-09T01:40:54,053 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38627 2024-12-09T01:40:54,057 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38627 2024-12-09T01:40:54,069 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38627 2024-12-09T01:40:54,070 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38627 2024-12-09T01:40:54,092 DEBUG [M:0;ef6f18c58dc9:37995 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ef6f18c58dc9:37995 2024-12-09T01:40:54,093 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ef6f18c58dc9,37995,1733708453073 2024-12-09T01:40:54,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38627-0x100746f49bc0001, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T01:40:54,103 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T01:40:54,105 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ef6f18c58dc9,37995,1733708453073 2024-12-09T01:40:54,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38627-0x100746f49bc0001, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T01:40:54,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:40:54,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38627-0x100746f49bc0001, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:40:54,131 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T01:40:54,134 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ef6f18c58dc9,37995,1733708453073 from backup master directory 2024-12-09T01:40:54,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ef6f18c58dc9,37995,1733708453073 2024-12-09T01:40:54,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T01:40:54,139 WARN [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T01:40:54,140 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ef6f18c58dc9,37995,1733708453073 2024-12-09T01:40:54,142 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T01:40:54,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38627-0x100746f49bc0001, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T01:40:54,144 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T01:40:54,212 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/hbase.id] with ID: 839c1de9-5c28-4534-95ef-dc24ebf74dc2 2024-12-09T01:40:54,212 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/.tmp/hbase.id 2024-12-09T01:40:54,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741826_1002 (size=42) 2024-12-09T01:40:54,626 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/.tmp/hbase.id]:[hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/hbase.id] 2024-12-09T01:40:54,679 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T01:40:54,688 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-09T01:40:54,712 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 22ms. 2024-12-09T01:40:54,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:40:54,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38627-0x100746f49bc0001, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:40:54,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741827_1003 (size=196) 2024-12-09T01:40:55,168 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:40:55,173 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T01:40:55,204 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:40:55,210 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T01:40:55,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741828_1004 (size=1189) 2024-12-09T01:40:55,682 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/data/master/store 2024-12-09T01:40:55,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741829_1005 (size=34) 2024-12-09T01:40:55,713 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T01:40:55,717 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:40:55,719 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T01:40:55,719 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:40:55,720 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:40:55,722 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T01:40:55,722 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:40:55,722 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:40:55,723 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733708455719Disabling compacts and flushes for region at 1733708455719Disabling writes for close at 1733708455722 (+3 ms)Writing region close event to WAL at 1733708455722Closed at 1733708455722 2024-12-09T01:40:55,726 WARN [master/ef6f18c58dc9:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/data/master/store/.initializing 2024-12-09T01:40:55,726 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/WALs/ef6f18c58dc9,37995,1733708453073 2024-12-09T01:40:55,735 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T01:40:55,752 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ef6f18c58dc9%2C37995%2C1733708453073, suffix=, logDir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/WALs/ef6f18c58dc9,37995,1733708453073, archiveDir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/oldWALs, maxLogs=10 2024-12-09T01:40:55,784 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/WALs/ef6f18c58dc9,37995,1733708453073/ef6f18c58dc9%2C37995%2C1733708453073.1733708455758, exclude list is [], retry=0 2024-12-09T01:40:55,804 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33243,DS-ef666e65-0c86-49cb-826c-7c6d3ccb7b06,DISK] 2024-12-09T01:40:55,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-09T01:40:55,851 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/WALs/ef6f18c58dc9,37995,1733708453073/ef6f18c58dc9%2C37995%2C1733708453073.1733708455758 2024-12-09T01:40:55,852 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39395:39395)] 2024-12-09T01:40:55,853 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:40:55,853 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:40:55,857 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:40:55,858 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:40:55,897 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:40:55,923 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T01:40:55,927 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:40:55,930 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:40:55,930 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:40:55,933 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T01:40:55,934 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:40:55,935 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:40:55,935 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:40:55,938 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T01:40:55,938 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:40:55,939 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:40:55,939 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:40:55,942 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T01:40:55,942 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:40:55,943 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:40:55,943 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:40:55,947 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:40:55,948 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:40:55,953 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:40:55,953 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:40:55,957 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T01:40:55,960 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T01:40:55,964 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:40:55,966 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59124347, jitterRate=-0.1189785748720169}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T01:40:55,973 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733708455871Initializing all the Stores at 1733708455873 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733708455874 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708455875 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708455875Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708455875Cleaning up temporary data from old regions at 1733708455953 (+78 ms)Region opened successfully at 1733708455972 (+19 ms) 2024-12-09T01:40:55,974 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T01:40:56,008 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d057b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef6f18c58dc9/172.17.0.2:0 2024-12-09T01:40:56,042 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-09T01:40:56,054 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T01:40:56,054 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T01:40:56,057 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T01:40:56,059 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-09T01:40:56,064 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-09T01:40:56,064 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T01:40:56,091 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T01:40:56,099 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T01:40:56,101 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-09T01:40:56,104 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T01:40:56,105 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T01:40:56,107 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-09T01:40:56,109 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T01:40:56,113 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T01:40:56,114 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-09T01:40:56,115 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T01:40:56,117 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T01:40:56,136 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T01:40:56,137 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T01:40:56,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38627-0x100746f49bc0001, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T01:40:56,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T01:40:56,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:40:56,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38627-0x100746f49bc0001, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:40:56,144 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ef6f18c58dc9,37995,1733708453073, sessionid=0x100746f49bc0000, setting cluster-up flag (Was=false) 2024-12-09T01:40:56,156 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:40:56,156 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38627-0x100746f49bc0001, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:40:56,163 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T01:40:56,165 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ef6f18c58dc9,37995,1733708453073 2024-12-09T01:40:56,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38627-0x100746f49bc0001, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:40:56,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:40:56,177 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T01:40:56,178 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ef6f18c58dc9,37995,1733708453073 2024-12-09T01:40:56,185 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-09T01:40:56,258 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-09T01:40:56,267 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-09T01:40:56,273 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T01:40:56,276 INFO [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer(746): ClusterId : 839c1de9-5c28-4534-95ef-dc24ebf74dc2 2024-12-09T01:40:56,279 DEBUG [RS:0;ef6f18c58dc9:38627 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T01:40:56,279 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ef6f18c58dc9,37995,1733708453073 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T01:40:56,285 DEBUG [RS:0;ef6f18c58dc9:38627 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T01:40:56,285 DEBUG [RS:0;ef6f18c58dc9:38627 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T01:40:56,286 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ef6f18c58dc9:0, corePoolSize=5, maxPoolSize=5 2024-12-09T01:40:56,286 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ef6f18c58dc9:0, corePoolSize=5, maxPoolSize=5 2024-12-09T01:40:56,287 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ef6f18c58dc9:0, corePoolSize=5, maxPoolSize=5 2024-12-09T01:40:56,287 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ef6f18c58dc9:0, corePoolSize=5, maxPoolSize=5 2024-12-09T01:40:56,287 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ef6f18c58dc9:0, corePoolSize=10, maxPoolSize=10 2024-12-09T01:40:56,287 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:40:56,287 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ef6f18c58dc9:0, corePoolSize=2, maxPoolSize=2 2024-12-09T01:40:56,288 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:40:56,288 DEBUG [RS:0;ef6f18c58dc9:38627 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T01:40:56,289 DEBUG [RS:0;ef6f18c58dc9:38627 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9d700d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ef6f18c58dc9/172.17.0.2:0 2024-12-09T01:40:56,294 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733708486294 2024-12-09T01:40:56,295 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T01:40:56,295 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-09T01:40:56,296 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T01:40:56,297 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T01:40:56,301 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T01:40:56,302 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T01:40:56,302 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T01:40:56,302 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T01:40:56,303 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:40:56,303 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T01:40:56,304 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T01:40:56,307 DEBUG [RS:0;ef6f18c58dc9:38627 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ef6f18c58dc9:38627 2024-12-09T01:40:56,308 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T01:40:56,310 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T01:40:56,310 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T01:40:56,313 INFO [RS:0;ef6f18c58dc9:38627 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-09T01:40:56,313 INFO [RS:0;ef6f18c58dc9:38627 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-09T01:40:56,313 DEBUG [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-09T01:40:56,316 INFO [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer(2659): reportForDuty to master=ef6f18c58dc9,37995,1733708453073 with port=38627, startcode=1733708453943 2024-12-09T01:40:56,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741831_1007 (size=1321) 2024-12-09T01:40:56,322 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-09T01:40:56,322 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb 2024-12-09T01:40:56,324 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T01:40:56,324 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T01:40:56,327 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ef6f18c58dc9:0:becomeActiveMaster-HFileCleaner.large.0-1733708456326,5,FailOnTimeoutGroup] 2024-12-09T01:40:56,328 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ef6f18c58dc9:0:becomeActiveMaster-HFileCleaner.small.0-1733708456327,5,FailOnTimeoutGroup] 2024-12-09T01:40:56,328 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T01:40:56,328 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T01:40:56,329 DEBUG [RS:0;ef6f18c58dc9:38627 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T01:40:56,330 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T01:40:56,330 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T01:40:56,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741832_1008 (size=32) 2024-12-09T01:40:56,340 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:40:56,344 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T01:40:56,347 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T01:40:56,347 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:40:56,350 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:40:56,350 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T01:40:56,353 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T01:40:56,353 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:40:56,354 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:40:56,355 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T01:40:56,360 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T01:40:56,360 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:40:56,362 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:40:56,362 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T01:40:56,364 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T01:40:56,364 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:40:56,365 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:40:56,366 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T01:40:56,367 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740 2024-12-09T01:40:56,368 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740 2024-12-09T01:40:56,371 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T01:40:56,371 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T01:40:56,372 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45211, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T01:40:56,372 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T01:40:56,375 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T01:40:56,380 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37995 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:56,381 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:40:56,382 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63299850, jitterRate=-0.056758731603622437}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T01:40:56,383 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37995 {}] master.ServerManager(517): Registering regionserver=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:56,386 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733708456341Initializing all the Stores at 1733708456343 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733708456343Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733708456344 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708456344Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733708456344Cleaning up temporary data from old regions at 1733708456371 (+27 ms)Region opened successfully at 1733708456386 (+15 ms) 2024-12-09T01:40:56,387 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T01:40:56,387 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T01:40:56,387 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T01:40:56,387 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T01:40:56,387 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T01:40:56,389 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T01:40:56,389 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733708456387Disabling compacts and flushes for region at 1733708456387Disabling writes for close at 1733708456387Writing region close event to WAL at 1733708456389 (+2 ms)Closed at 1733708456389 2024-12-09T01:40:56,393 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T01:40:56,394 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-09T01:40:56,400 DEBUG [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb 2024-12-09T01:40:56,400 DEBUG [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40841 2024-12-09T01:40:56,400 DEBUG [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-09T01:40:56,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T01:40:56,405 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T01:40:56,406 DEBUG [RS:0;ef6f18c58dc9:38627 {}] zookeeper.ZKUtil(111): regionserver:38627-0x100746f49bc0001, quorum=127.0.0.1:50027, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:56,406 WARN [RS:0;ef6f18c58dc9:38627 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T01:40:56,406 INFO [RS:0;ef6f18c58dc9:38627 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T01:40:56,406 DEBUG [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/WALs/ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:56,407 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ef6f18c58dc9,38627,1733708453943] 2024-12-09T01:40:56,413 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T01:40:56,418 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T01:40:56,433 INFO [RS:0;ef6f18c58dc9:38627 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T01:40:56,452 INFO [RS:0;ef6f18c58dc9:38627 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T01:40:56,459 INFO [RS:0;ef6f18c58dc9:38627 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T01:40:56,459 INFO [RS:0;ef6f18c58dc9:38627 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:40:56,460 INFO [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-09T01:40:56,466 INFO [RS:0;ef6f18c58dc9:38627 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-09T01:40:56,468 INFO [RS:0;ef6f18c58dc9:38627 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T01:40:56,468 DEBUG [RS:0;ef6f18c58dc9:38627 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:40:56,468 DEBUG [RS:0;ef6f18c58dc9:38627 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:40:56,468 DEBUG [RS:0;ef6f18c58dc9:38627 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:40:56,468 DEBUG [RS:0;ef6f18c58dc9:38627 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:40:56,469 DEBUG [RS:0;ef6f18c58dc9:38627 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:40:56,469 DEBUG [RS:0;ef6f18c58dc9:38627 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ef6f18c58dc9:0, corePoolSize=2, maxPoolSize=2 2024-12-09T01:40:56,469 DEBUG [RS:0;ef6f18c58dc9:38627 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:40:56,469 DEBUG [RS:0;ef6f18c58dc9:38627 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:40:56,469 DEBUG [RS:0;ef6f18c58dc9:38627 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:40:56,469 DEBUG [RS:0;ef6f18c58dc9:38627 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:40:56,469 DEBUG [RS:0;ef6f18c58dc9:38627 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:40:56,469 DEBUG [RS:0;ef6f18c58dc9:38627 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ef6f18c58dc9:0, corePoolSize=1, maxPoolSize=1 2024-12-09T01:40:56,470 DEBUG [RS:0;ef6f18c58dc9:38627 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ef6f18c58dc9:0, corePoolSize=3, maxPoolSize=3 2024-12-09T01:40:56,470 DEBUG [RS:0;ef6f18c58dc9:38627 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0, corePoolSize=3, maxPoolSize=3 2024-12-09T01:40:56,471 INFO [RS:0;ef6f18c58dc9:38627 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T01:40:56,471 INFO [RS:0;ef6f18c58dc9:38627 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T01:40:56,471 INFO [RS:0;ef6f18c58dc9:38627 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:40:56,471 INFO [RS:0;ef6f18c58dc9:38627 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T01:40:56,471 INFO [RS:0;ef6f18c58dc9:38627 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T01:40:56,472 INFO [RS:0;ef6f18c58dc9:38627 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,38627,1733708453943-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T01:40:56,491 INFO [RS:0;ef6f18c58dc9:38627 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T01:40:56,493 INFO [RS:0;ef6f18c58dc9:38627 {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,38627,1733708453943-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:40:56,494 INFO [RS:0;ef6f18c58dc9:38627 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:40:56,494 INFO [RS:0;ef6f18c58dc9:38627 {}] regionserver.Replication(171): ef6f18c58dc9,38627,1733708453943 started 2024-12-09T01:40:56,514 INFO [RS:0;ef6f18c58dc9:38627 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:40:56,514 INFO [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer(1482): Serving as ef6f18c58dc9,38627,1733708453943, RpcServer on ef6f18c58dc9/172.17.0.2:38627, sessionid=0x100746f49bc0001 2024-12-09T01:40:56,515 DEBUG [RS:0;ef6f18c58dc9:38627 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T01:40:56,515 DEBUG [RS:0;ef6f18c58dc9:38627 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:56,516 DEBUG [RS:0;ef6f18c58dc9:38627 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef6f18c58dc9,38627,1733708453943' 2024-12-09T01:40:56,516 DEBUG [RS:0;ef6f18c58dc9:38627 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T01:40:56,517 DEBUG [RS:0;ef6f18c58dc9:38627 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T01:40:56,518 DEBUG [RS:0;ef6f18c58dc9:38627 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T01:40:56,518 DEBUG [RS:0;ef6f18c58dc9:38627 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T01:40:56,518 DEBUG [RS:0;ef6f18c58dc9:38627 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:56,518 DEBUG [RS:0;ef6f18c58dc9:38627 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ef6f18c58dc9,38627,1733708453943' 2024-12-09T01:40:56,518 DEBUG [RS:0;ef6f18c58dc9:38627 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T01:40:56,519 DEBUG [RS:0;ef6f18c58dc9:38627 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T01:40:56,520 DEBUG [RS:0;ef6f18c58dc9:38627 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T01:40:56,520 INFO [RS:0;ef6f18c58dc9:38627 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T01:40:56,520 INFO [RS:0;ef6f18c58dc9:38627 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T01:40:56,569 WARN [ef6f18c58dc9:37995 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-09T01:40:56,626 INFO [RS:0;ef6f18c58dc9:38627 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T01:40:56,630 INFO [RS:0;ef6f18c58dc9:38627 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ef6f18c58dc9%2C38627%2C1733708453943, suffix=, logDir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/WALs/ef6f18c58dc9,38627,1733708453943, archiveDir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/oldWALs, maxLogs=32 2024-12-09T01:40:56,647 DEBUG [RS:0;ef6f18c58dc9:38627 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/WALs/ef6f18c58dc9,38627,1733708453943/ef6f18c58dc9%2C38627%2C1733708453943.1733708456632, exclude list is [], retry=0 2024-12-09T01:40:56,652 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33243,DS-ef666e65-0c86-49cb-826c-7c6d3ccb7b06,DISK] 2024-12-09T01:40:56,657 INFO [RS:0;ef6f18c58dc9:38627 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/WALs/ef6f18c58dc9,38627,1733708453943/ef6f18c58dc9%2C38627%2C1733708453943.1733708456632 2024-12-09T01:40:56,659 DEBUG [RS:0;ef6f18c58dc9:38627 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39395:39395)] 2024-12-09T01:40:56,822 DEBUG [ef6f18c58dc9:37995 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-09T01:40:56,834 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:56,841 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ef6f18c58dc9,38627,1733708453943, state=OPENING 2024-12-09T01:40:56,846 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T01:40:56,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38627-0x100746f49bc0001, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:40:56,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:40:56,849 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T01:40:56,849 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T01:40:56,850 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T01:40:56,852 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ef6f18c58dc9,38627,1733708453943}] 2024-12-09T01:40:57,033 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T01:40:57,039 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52215, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T01:40:57,055 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-09T01:40:57,055 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T01:40:57,056 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T01:40:57,061 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ef6f18c58dc9%2C38627%2C1733708453943.meta, suffix=.meta, logDir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/WALs/ef6f18c58dc9,38627,1733708453943, archiveDir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/oldWALs, maxLogs=32 2024-12-09T01:40:57,086 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/WALs/ef6f18c58dc9,38627,1733708453943/ef6f18c58dc9%2C38627%2C1733708453943.meta.1733708457063.meta, exclude list is [], retry=0 2024-12-09T01:40:57,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33243,DS-ef666e65-0c86-49cb-826c-7c6d3ccb7b06,DISK] 2024-12-09T01:40:57,095 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/WALs/ef6f18c58dc9,38627,1733708453943/ef6f18c58dc9%2C38627%2C1733708453943.meta.1733708457063.meta 2024-12-09T01:40:57,095 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39395:39395)] 2024-12-09T01:40:57,096 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:40:57,098 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T01:40:57,101 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T01:40:57,107 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T01:40:57,113 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T01:40:57,114 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:40:57,115 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-09T01:40:57,115 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-09T01:40:57,119 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T01:40:57,121 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T01:40:57,121 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:40:57,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:40:57,122 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-09T01:40:57,123 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-09T01:40:57,123 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:40:57,124 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:40:57,124 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T01:40:57,126 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T01:40:57,126 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:40:57,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:40:57,127 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T01:40:57,128 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T01:40:57,128 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:40:57,129 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T01:40:57,129 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-09T01:40:57,131 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740 2024-12-09T01:40:57,133 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740 2024-12-09T01:40:57,136 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-09T01:40:57,136 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-09T01:40:57,137 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T01:40:57,139 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-09T01:40:57,141 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67240670, jitterRate=0.001964062452316284}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T01:40:57,141 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-09T01:40:57,142 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733708457116Writing region info on filesystem at 1733708457116Initializing all the Stores at 1733708457118 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733708457118Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733708457119 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708457119Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733708457119Cleaning up temporary data from old regions at 1733708457136 (+17 ms)Running coprocessor post-open hooks at 1733708457141 (+5 ms)Region opened successfully at 1733708457142 (+1 ms) 2024-12-09T01:40:57,150 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733708457022 2024-12-09T01:40:57,162 DEBUG [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T01:40:57,162 INFO [RS_OPEN_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-09T01:40:57,164 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:57,166 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ef6f18c58dc9,38627,1733708453943, state=OPEN 2024-12-09T01:40:57,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38627-0x100746f49bc0001, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T01:40:57,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T01:40:57,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T01:40:57,172 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T01:40:57,172 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:57,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T01:40:57,177 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ef6f18c58dc9,38627,1733708453943 in 321 msec 2024-12-09T01:40:57,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T01:40:57,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 777 msec 2024-12-09T01:40:57,185 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-09T01:40:57,185 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-09T01:40:57,209 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:40:57,210 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:40:57,234 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:40:57,237 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56631, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:40:57,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0440 sec 2024-12-09T01:40:57,258 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733708457258, completionTime=-1 2024-12-09T01:40:57,260 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-09T01:40:57,260 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-09T01:40:57,320 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-09T01:40:57,320 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733708517320 2024-12-09T01:40:57,320 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733708577320 2024-12-09T01:40:57,320 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 59 msec 2024-12-09T01:40:57,323 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,37995,1733708453073-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T01:40:57,324 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,37995,1733708453073-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:40:57,324 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,37995,1733708453073-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:40:57,325 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ef6f18c58dc9:37995, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:40:57,326 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T01:40:57,327 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-09T01:40:57,338 DEBUG [master/ef6f18c58dc9:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-09T01:40:57,360 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.220sec 2024-12-09T01:40:57,361 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T01:40:57,362 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T01:40:57,363 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T01:40:57,364 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T01:40:57,364 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T01:40:57,365 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,37995,1733708453073-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T01:40:57,365 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,37995,1733708453073-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T01:40:57,373 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-09T01:40:57,374 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T01:40:57,374 INFO [master/ef6f18c58dc9:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ef6f18c58dc9,37995,1733708453073-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T01:40:57,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e3e3753, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:57,393 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T01:40:57,393 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T01:40:57,397 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:40:57,400 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:40:57,408 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:40:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:40:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:40:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fd4d87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:57,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:40:57,413 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:40:57,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:40:57,417 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:40:57,419 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35f487a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:57,420 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:40:57,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:40:57,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:40:57,430 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:40:57,450 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ef6f18c58dc9,37995,1733708453073 2024-12-09T01:40:57,462 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=446, ProcessCount=11, AvailableMemoryMB=12528 2024-12-09T01:40:57,471 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T01:40:57,476 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ef6f18c58dc9,37995,1733708453073 2024-12-09T01:40:57,479 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@219792a1 2024-12-09T01:40:57,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T01:40:57,482 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T01:40:57,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T01:40:57,489 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:40:57,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T01:40:57,500 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T01:40:57,503 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-12-09T01:40:57,504 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:40:57,507 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T01:40:57,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T01:40:57,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741835_1011 (size=1061) 2024-12-09T01:40:57,524 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb 2024-12-09T01:40:57,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741836_1012 (size=53) 2024-12-09T01:40:57,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T01:40:57,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T01:40:57,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:40:57,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 88980a4de80c5830889346818652c3a7, disabling compactions & flushes 2024-12-09T01:40:57,934 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:57,934 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:57,935 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. after waiting 0 ms 2024-12-09T01:40:57,935 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:57,935 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:57,935 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 88980a4de80c5830889346818652c3a7: Waiting for close lock at 1733708457934Disabling compacts and flushes for region at 1733708457934Disabling writes for close at 1733708457935 (+1 ms)Writing region close event to WAL at 1733708457935Closed at 1733708457935 2024-12-09T01:40:57,938 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T01:40:57,943 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733708457938"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733708457938"}]},"ts":"1733708457938"} 2024-12-09T01:40:57,948 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T01:40:57,950 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T01:40:57,953 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708457950"}]},"ts":"1733708457950"} 2024-12-09T01:40:57,957 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T01:40:57,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88980a4de80c5830889346818652c3a7, ASSIGN}] 2024-12-09T01:40:57,961 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88980a4de80c5830889346818652c3a7, ASSIGN 2024-12-09T01:40:57,963 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=88980a4de80c5830889346818652c3a7, ASSIGN; state=OFFLINE, location=ef6f18c58dc9,38627,1733708453943; forceNewPlan=false, retain=false 2024-12-09T01:40:58,114 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=88980a4de80c5830889346818652c3a7, regionState=OPENING, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:58,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=88980a4de80c5830889346818652c3a7, ASSIGN because future has completed 2024-12-09T01:40:58,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943}] 2024-12-09T01:40:58,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T01:40:58,281 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:58,281 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:40:58,282 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 88980a4de80c5830889346818652c3a7 2024-12-09T01:40:58,282 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:40:58,282 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 88980a4de80c5830889346818652c3a7 2024-12-09T01:40:58,282 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 88980a4de80c5830889346818652c3a7 2024-12-09T01:40:58,284 INFO [StoreOpener-88980a4de80c5830889346818652c3a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 88980a4de80c5830889346818652c3a7 2024-12-09T01:40:58,288 INFO [StoreOpener-88980a4de80c5830889346818652c3a7-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:40:58,289 INFO [StoreOpener-88980a4de80c5830889346818652c3a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88980a4de80c5830889346818652c3a7 columnFamilyName A 2024-12-09T01:40:58,289 DEBUG [StoreOpener-88980a4de80c5830889346818652c3a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:40:58,290 INFO [StoreOpener-88980a4de80c5830889346818652c3a7-1 {}] regionserver.HStore(327): Store=88980a4de80c5830889346818652c3a7/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:40:58,290 INFO [StoreOpener-88980a4de80c5830889346818652c3a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 88980a4de80c5830889346818652c3a7 2024-12-09T01:40:58,292 INFO [StoreOpener-88980a4de80c5830889346818652c3a7-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:40:58,293 INFO [StoreOpener-88980a4de80c5830889346818652c3a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88980a4de80c5830889346818652c3a7 columnFamilyName B 2024-12-09T01:40:58,293 DEBUG [StoreOpener-88980a4de80c5830889346818652c3a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:40:58,294 INFO [StoreOpener-88980a4de80c5830889346818652c3a7-1 {}] regionserver.HStore(327): Store=88980a4de80c5830889346818652c3a7/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:40:58,294 INFO [StoreOpener-88980a4de80c5830889346818652c3a7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 88980a4de80c5830889346818652c3a7 2024-12-09T01:40:58,296 INFO [StoreOpener-88980a4de80c5830889346818652c3a7-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:40:58,297 INFO [StoreOpener-88980a4de80c5830889346818652c3a7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88980a4de80c5830889346818652c3a7 columnFamilyName C 2024-12-09T01:40:58,297 DEBUG [StoreOpener-88980a4de80c5830889346818652c3a7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:40:58,298 INFO [StoreOpener-88980a4de80c5830889346818652c3a7-1 {}] regionserver.HStore(327): Store=88980a4de80c5830889346818652c3a7/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:40:58,298 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:58,298 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 88980a4de80c5830889346818652c3a7 2024-12-09T01:40:58,299 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7 2024-12-09T01:40:58,300 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7 2024-12-09T01:40:58,302 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 88980a4de80c5830889346818652c3a7 2024-12-09T01:40:58,302 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 88980a4de80c5830889346818652c3a7 2024-12-09T01:40:58,303 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T01:40:58,306 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 88980a4de80c5830889346818652c3a7 2024-12-09T01:40:58,310 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:40:58,311 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 88980a4de80c5830889346818652c3a7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66264440, jitterRate=-0.012582898139953613}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T01:40:58,311 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88980a4de80c5830889346818652c3a7 2024-12-09T01:40:58,312 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 88980a4de80c5830889346818652c3a7: Running coprocessor pre-open hook at 1733708458282Writing region info on filesystem at 1733708458282Initializing all the Stores at 1733708458284 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708458284Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708458284Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708458284Cleaning up temporary data from old regions at 1733708458302 (+18 ms)Running coprocessor post-open hooks at 1733708458311 (+9 ms)Region opened successfully at 1733708458312 (+1 ms) 2024-12-09T01:40:58,314 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., pid=6, masterSystemTime=1733708458274 2024-12-09T01:40:58,318 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:58,318 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:58,320 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=88980a4de80c5830889346818652c3a7, regionState=OPEN, openSeqNum=2, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:58,323 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 because future has completed 2024-12-09T01:40:58,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T01:40:58,329 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 in 205 msec 2024-12-09T01:40:58,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T01:40:58,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88980a4de80c5830889346818652c3a7, ASSIGN in 370 msec 2024-12-09T01:40:58,334 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T01:40:58,334 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708458334"}]},"ts":"1733708458334"} 2024-12-09T01:40:58,337 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T01:40:58,338 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T01:40:58,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 846 msec 2024-12-09T01:40:58,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-09T01:40:58,645 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:40:58,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ed391d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,651 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:40:58,652 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:40:58,653 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:40:58,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:40:58,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:40:58,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ea8828e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:40:58,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:40:58,655 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:40:58,656 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:40:58,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d2b9a42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,658 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:40:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:40:58,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:40:58,662 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:40:58,665 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T01:40:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is ef6f18c58dc9,37995,1733708453073 2024-12-09T01:40:58,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@a040aee 2024-12-09T01:40:58,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T01:40:58,671 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58668, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T01:40:58,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e6c3037, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,675 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:40:58,675 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:40:58,677 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:40:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:40:58,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:40:58,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dd81cbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:40:58,678 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:40:58,679 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:40:58,680 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:40:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f9a6e4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a519624, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,682 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:40:58,683 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:40:58,684 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:40:58,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:40:58,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:40:58,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@89c0433, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:40:58,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:40:58,685 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:40:58,687 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58694, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:40:58,688 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ef39c97, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,688 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aa662a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,689 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:40:58,689 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:40:58,691 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:40:58,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:40:58,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:40:58,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a7f3cf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:40:58,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:40:58,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:40:58,694 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:40:58,695 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@540312d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a38e5f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,696 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:40:58,696 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:40:58,698 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:40:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:40:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:40:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61ee18a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,698 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:40:58,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:40:58,699 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:40:58,701 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58732, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:40:58,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@256affd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,702 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@658d066f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,703 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:40:58,703 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:40:58,704 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:40:58,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:40:58,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:40:58,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b46e875, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:40:58,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:40:58,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:40:58,706 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:40:58,707 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56b466e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d220de4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,708 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:40:58,709 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:40:58,709 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:40:58,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:40:58,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:40:58,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f1b5c7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:40:58,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:40:58,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:40:58,712 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58762, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:40:58,713 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5124603b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,714 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32cda1eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,714 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:40:58,714 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:40:58,715 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:40:58,715 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:40:58,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:40:58,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1290d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:40:58,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:40:58,716 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:40:58,717 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58780, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:40:58,718 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38d57b05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,719 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77d59c73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,719 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:40:58,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:40:58,721 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:40:58,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:40:58,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:40:58,721 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67a7f0f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:40:58,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:40:58,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:40:58,723 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:40:58,724 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48230300, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,725 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27451c77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,725 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:40:58,725 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:40:58,726 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:40:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:40:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:40:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2268467d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:40:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:40:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:40:58,728 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:40:58,729 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0f145b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:40:58,733 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:40:58,733 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:40:58,733 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:40:58,734 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:40:58,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:40:58,735 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:40:58,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:40:58,736 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:40:58,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:40:58,736 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:40:58,736 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:40:58,737 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:40:58,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34736, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:40:58,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:40:58,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:40:58,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:40:58,739 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:40:58,739 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:40:58,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:40:58,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:40:58,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34762, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:40:58,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:40:58,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:40:58,743 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:40:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-12-09T01:40:58,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:40:58,747 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:40:58,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:40:58,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:40:58,749 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:40:58,749 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:40:58,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:40:58,751 DEBUG [Thread-147 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:40:58,751 DEBUG [Thread-147 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:40:58,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:40:58,755 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:40:58,755 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:40:58,760 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:40:58,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:40:58,762 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:40:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T01:40:58,768 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:40:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:40:58,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:40:58,772 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:40:58,772 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:40:58,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:40:58,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:40:58,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:40:58,776 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34796, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:40:58,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:40:58,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:40:58,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T01:40:58,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:40:58,857 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:40:58,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:40:58,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:40:58,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:40:58,858 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:40:58,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T01:40:58,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T01:40:58,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:58,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:40:58,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:58,961 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:40:58,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:40:58,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/e5fe0ca26d454dad92cb4167e14f4a35 is 50, key is test_row_0/A:col10/1733708458831/Put/seqid=0 2024-12-09T01:40:58,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:40:58,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:40:59,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708518964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:59,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708458943, max timestamp=1733708458968Num uniques 859; 2024-12-09T01:40:59,009 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:40:59,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708458943, max timestamp=1733708458978Num uniques 859; 2024-12-09T01:40:59,026 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:40:59,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733708458943, max timestamp=1733708458968Num uniques 859; 2024-12-09T01:40:59,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T01:40:59,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741837_1013 (size=9657) 2024-12-09T01:40:59,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/e5fe0ca26d454dad92cb4167e14f4a35 2024-12-09T01:40:59,047 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:40:59,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T01:40:59,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:40:59,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708519100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:59,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:40:59,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,125 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:40:59,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708519104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:40:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:40:59,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:40:59,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708519116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:59,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708519112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:59,128 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:40:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:40:59,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T01:40:59,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:40:59,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708519155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:59,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:40:59,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:59,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:40:59,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:59,176 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:40:59,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:40:59,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:40:59,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/88599adc073c4d508ad3786e96bd1a42 is 50, key is test_row_0/B:col10/1733708458831/Put/seqid=0 2024-12-09T01:40:59,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741838_1014 (size=9657) 2024-12-09T01:40:59,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/88599adc073c4d508ad3786e96bd1a42 2024-12-09T01:40:59,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:40:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708519236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:59,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:40:59,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:40:59,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708519238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:59,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:40:59,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708519237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:59,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:40:59,240 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708519238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:59,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:40:59,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:40:59,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:40:59,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/0d7548d0de7b45aa9446f8af2525f358 is 50, key is test_row_0/C:col10/1733708458831/Put/seqid=0 2024-12-09T01:40:59,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741839_1015 (size=9657) 2024-12-09T01:40:59,290 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/0d7548d0de7b45aa9446f8af2525f358 2024-12-09T01:40:59,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/e5fe0ca26d454dad92cb4167e14f4a35 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/e5fe0ca26d454dad92cb4167e14f4a35 2024-12-09T01:40:59,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/e5fe0ca26d454dad92cb4167e14f4a35, entries=100, sequenceid=12, filesize=9.4 K 2024-12-09T01:40:59,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/88599adc073c4d508ad3786e96bd1a42 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/88599adc073c4d508ad3786e96bd1a42 2024-12-09T01:40:59,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T01:40:59,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:59,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:40:59,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:59,338 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:40:59,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:40:59,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/88599adc073c4d508ad3786e96bd1a42, entries=100, sequenceid=12, filesize=9.4 K 2024-12-09T01:40:59,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:40:59,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/0d7548d0de7b45aa9446f8af2525f358 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/0d7548d0de7b45aa9446f8af2525f358 2024-12-09T01:40:59,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/0d7548d0de7b45aa9446f8af2525f358, entries=100, sequenceid=12, filesize=9.4 K 2024-12-09T01:40:59,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.08 KB/130125 for 88980a4de80c5830889346818652c3a7 in 515ms, sequenceid=12, compaction requested=false 2024-12-09T01:40:59,367 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-09T01:40:59,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:40:59,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:40:59,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=133.78 KB heapSize=384.07 KB 2024-12-09T01:40:59,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:40:59,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:40:59,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:40:59,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:40:59,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:40:59,393 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:40:59,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T01:40:59,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/c4f4e71da53b4be3a8538e1c9389a283 is 50, key is test_row_0/A:col10/1733708459098/Put/seqid=0 2024-12-09T01:40:59,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741840_1016 (size=14341) 2024-12-09T01:40:59,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/c4f4e71da53b4be3a8538e1c9389a283 2024-12-09T01:40:59,486 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/8bdd8ae326e74e50a4ca21e0c01bfc01 is 50, key is test_row_0/B:col10/1733708459098/Put/seqid=0 2024-12-09T01:40:59,496 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T01:40:59,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:59,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:40:59,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:59,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:40:59,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:40:59,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:40:59,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:40:59,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708519482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:59,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,508 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:40:59,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741841_1017 (size=12001) 2024-12-09T01:40:59,515 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/8bdd8ae326e74e50a4ca21e0c01bfc01 2024-12-09T01:40:59,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:40:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708519489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:59,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,538 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:40:59,539 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:40:59,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708519491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:59,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,545 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:40:59,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:40:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708519506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:59,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/afcd5d9c94424b11bf71a32aac172351 is 50, key is test_row_0/C:col10/1733708459098/Put/seqid=0 2024-12-09T01:40:59,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:40:59,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:40:59,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708519506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:59,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,556 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:40:59,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741842_1018 (size=12001) 2024-12-09T01:40:59,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/afcd5d9c94424b11bf71a32aac172351 2024-12-09T01:40:59,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/c4f4e71da53b4be3a8538e1c9389a283 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/c4f4e71da53b4be3a8538e1c9389a283 2024-12-09T01:40:59,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/c4f4e71da53b4be3a8538e1c9389a283, entries=200, sequenceid=49, filesize=14.0 K 2024-12-09T01:40:59,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/8bdd8ae326e74e50a4ca21e0c01bfc01 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/8bdd8ae326e74e50a4ca21e0c01bfc01 2024-12-09T01:40:59,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:40:59,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708519614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:59,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:40:59,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/8bdd8ae326e74e50a4ca21e0c01bfc01, entries=150, sequenceid=49, filesize=11.7 K 2024-12-09T01:40:59,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/afcd5d9c94424b11bf71a32aac172351 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/afcd5d9c94424b11bf71a32aac172351 2024-12-09T01:40:59,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T01:40:59,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:59,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:40:59,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:59,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:40:59,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:40:59,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:40:59,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/afcd5d9c94424b11bf71a32aac172351, entries=150, sequenceid=49, filesize=11.7 K 2024-12-09T01:40:59,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~133.78 KB/136995, heapSize ~384.02 KB/393240, currentSize=46.96 KB/48090 for 88980a4de80c5830889346818652c3a7 in 273ms, sequenceid=49, compaction requested=false 2024-12-09T01:40:59,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:40:59,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:40:59,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T01:40:59,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:40:59,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:40:59,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:40:59,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:40:59,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:40:59,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:40:59,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/37b12b31e923443db3bef1bfd3478581 is 50, key is test_row_0/A:col10/1733708459488/Put/seqid=0 2024-12-09T01:40:59,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741843_1019 (size=14341) 2024-12-09T01:40:59,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/37b12b31e923443db3bef1bfd3478581 2024-12-09T01:40:59,737 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/fb183aab70a847cf86f7c4281d296add is 50, key is test_row_0/B:col10/1733708459488/Put/seqid=0 2024-12-09T01:40:59,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741844_1020 (size=12001) 2024-12-09T01:40:59,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/fb183aab70a847cf86f7c4281d296add 2024-12-09T01:40:59,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/f1415d388db04dd781f192525e5abac5 is 50, key is test_row_0/C:col10/1733708459488/Put/seqid=0 2024-12-09T01:40:59,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T01:40:59,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:59,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:40:59,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:59,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:40:59,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:40:59,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:40:59,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741845_1021 (size=12001) 2024-12-09T01:40:59,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/f1415d388db04dd781f192525e5abac5 2024-12-09T01:40:59,823 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708459788, max timestamp=1733708459813Num uniques 859; 2024-12-09T01:40:59,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:40:59,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/37b12b31e923443db3bef1bfd3478581 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/37b12b31e923443db3bef1bfd3478581 2024-12-09T01:40:59,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733708459788, max timestamp=1733708459813Num uniques 859; 2024-12-09T01:40:59,847 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T01:40:59,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708459788, max timestamp=1733708459817Num uniques 859; 2024-12-09T01:40:59,848 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:40:59,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/37b12b31e923443db3bef1bfd3478581, entries=200, sequenceid=60, filesize=14.0 K 2024-12-09T01:40:59,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/fb183aab70a847cf86f7c4281d296add as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/fb183aab70a847cf86f7c4281d296add 2024-12-09T01:40:59,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/fb183aab70a847cf86f7c4281d296add, entries=150, sequenceid=60, filesize=11.7 K 2024-12-09T01:40:59,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/f1415d388db04dd781f192525e5abac5 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/f1415d388db04dd781f192525e5abac5 2024-12-09T01:40:59,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/f1415d388db04dd781f192525e5abac5, entries=150, sequenceid=60, filesize=11.7 K 2024-12-09T01:40:59,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=89.06 KB/91195 for 88980a4de80c5830889346818652c3a7 in 216ms, sequenceid=60, compaction requested=true 2024-12-09T01:40:59,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:40:59,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:40:59,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:40:59,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T01:40:59,905 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:40:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:40:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:40:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:40:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:40:59,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=109.18 KB heapSize=309.46 KB 2024-12-09T01:40:59,907 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:40:59,907 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:40:59,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:40:59,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:40:59,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:40:59,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:40:59,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:40:59,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:40:59,912 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:40:59,912 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38339 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:40:59,914 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/B is initiating minor compaction (all files) 2024-12-09T01:40:59,915 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/B in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:59,915 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/88599adc073c4d508ad3786e96bd1a42, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/8bdd8ae326e74e50a4ca21e0c01bfc01, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/fb183aab70a847cf86f7c4281d296add] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=32.9 K 2024-12-09T01:40:59,917 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 88599adc073c4d508ad3786e96bd1a42, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733708458757 2024-12-09T01:40:59,918 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bdd8ae326e74e50a4ca21e0c01bfc01, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733708459098 2024-12-09T01:40:59,919 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting fb183aab70a847cf86f7c4281d296add, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733708459488 2024-12-09T01:40:59,923 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/A is initiating minor compaction (all files) 2024-12-09T01:40:59,924 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/A in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:59,924 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/e5fe0ca26d454dad92cb4167e14f4a35, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/c4f4e71da53b4be3a8538e1c9389a283, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/37b12b31e923443db3bef1bfd3478581] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=37.4 K 2024-12-09T01:40:59,925 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5fe0ca26d454dad92cb4167e14f4a35, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733708458757 2024-12-09T01:40:59,926 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4f4e71da53b4be3a8538e1c9389a283, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733708459065 2024-12-09T01:40:59,927 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37b12b31e923443db3bef1bfd3478581, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733708459463 2024-12-09T01:40:59,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/9e344e5757c2476fbc617b851fabc862 is 50, key is test_row_0/A:col10/1733708459900/Put/seqid=0 2024-12-09T01:40:59,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T01:40:59,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:59,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:40:59,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:40:59,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:40:59,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:40:59,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:40:59,978 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#B#compaction#10 average throughput is 0.33 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:40:59,979 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/dc13c71485f4497c9f6c28a184dc7efe is 50, key is test_row_0/B:col10/1733708459488/Put/seqid=0 2024-12-09T01:40:59,982 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#A#compaction#11 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:40:59,983 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/ef564974680248f0a429d59e0d430cba is 50, key is test_row_0/A:col10/1733708459488/Put/seqid=0 2024-12-09T01:40:59,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:40:59,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708519964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:40:59,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:40:59,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:40:59,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741846_1022 (size=16681) 2024-12-09T01:41:00,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708519975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,007 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708519979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,010 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708519983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/9e344e5757c2476fbc617b851fabc862 2024-12-09T01:41:00,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708519986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,013 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741847_1023 (size=12104) 2024-12-09T01:41:00,055 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/dc13c71485f4497c9f6c28a184dc7efe as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/dc13c71485f4497c9f6c28a184dc7efe 2024-12-09T01:41:00,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741848_1024 (size=12104) 2024-12-09T01:41:00,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/7c94c8d74a334bdb85821b5311af04c1 is 50, key is test_row_0/B:col10/1733708459900/Put/seqid=0 2024-12-09T01:41:00,076 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/ef564974680248f0a429d59e0d430cba as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ef564974680248f0a429d59e0d430cba 2024-12-09T01:41:00,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708520096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,098 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/A of 88980a4de80c5830889346818652c3a7 into ef564974680248f0a429d59e0d430cba(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:00,098 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,100 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/A, priority=13, startTime=1733708459892; duration=0sec 2024-12-09T01:41:00,101 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:00,101 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:A 2024-12-09T01:41:00,101 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:00,101 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/B of 88980a4de80c5830889346818652c3a7 into dc13c71485f4497c9f6c28a184dc7efe(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:00,102 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:00,102 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/B, priority=13, startTime=1733708459905; duration=0sec 2024-12-09T01:41:00,102 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:00,102 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:B 2024-12-09T01:41:00,104 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:00,104 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/C is initiating minor compaction (all files) 2024-12-09T01:41:00,104 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/C in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:00,104 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/0d7548d0de7b45aa9446f8af2525f358, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/afcd5d9c94424b11bf71a32aac172351, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/f1415d388db04dd781f192525e5abac5] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=32.9 K 2024-12-09T01:41:00,105 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d7548d0de7b45aa9446f8af2525f358, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1733708458757 2024-12-09T01:41:00,106 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting afcd5d9c94424b11bf71a32aac172351, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1733708459098 2024-12-09T01:41:00,107 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1415d388db04dd781f192525e5abac5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733708459488 2024-12-09T01:41:00,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741849_1025 (size=12001) 2024-12-09T01:41:00,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=37.62 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/7c94c8d74a334bdb85821b5311af04c1 2024-12-09T01:41:00,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T01:41:00,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:00,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:00,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:00,123 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:00,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:00,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708520116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708520116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708520116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708520124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,142 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#C#compaction#13 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:00,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/48a6ef2782574928845a2761d501354e is 50, key is test_row_0/C:col10/1733708459900/Put/seqid=0 2024-12-09T01:41:00,153 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/5cb06c2dcb524d1995af449df58877bd is 50, key is test_row_0/C:col10/1733708459488/Put/seqid=0 2024-12-09T01:41:00,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741850_1026 (size=12001) 2024-12-09T01:41:00,169 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/48a6ef2782574928845a2761d501354e 2024-12-09T01:41:00,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/9e344e5757c2476fbc617b851fabc862 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/9e344e5757c2476fbc617b851fabc862 2024-12-09T01:41:00,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741851_1027 (size=12104) 2024-12-09T01:41:00,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/9e344e5757c2476fbc617b851fabc862, entries=250, sequenceid=94, filesize=16.3 K 2024-12-09T01:41:00,206 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/5cb06c2dcb524d1995af449df58877bd as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/5cb06c2dcb524d1995af449df58877bd 2024-12-09T01:41:00,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/7c94c8d74a334bdb85821b5311af04c1 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/7c94c8d74a334bdb85821b5311af04c1 2024-12-09T01:41:00,225 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/C of 88980a4de80c5830889346818652c3a7 into 5cb06c2dcb524d1995af449df58877bd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:00,225 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:00,225 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/C, priority=13, startTime=1733708459906; duration=0sec 2024-12-09T01:41:00,225 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:00,225 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:C 2024-12-09T01:41:00,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/7c94c8d74a334bdb85821b5311af04c1, entries=150, sequenceid=94, filesize=11.7 K 2024-12-09T01:41:00,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/48a6ef2782574928845a2761d501354e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/48a6ef2782574928845a2761d501354e 2024-12-09T01:41:00,247 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/48a6ef2782574928845a2761d501354e, entries=150, sequenceid=94, filesize=11.7 K 2024-12-09T01:41:00,249 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~115.89 KB/118675, heapSize ~329.34 KB/337240, currentSize=67.09 KB/68700 for 88980a4de80c5830889346818652c3a7 in 343ms, sequenceid=94, compaction requested=false 2024-12-09T01:41:00,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:00,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-12-09T01:41:00,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:00,281 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T01:41:00,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:00,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:00,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:00,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:00,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:00,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:00,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/8006b6bbf9904dd18d58061214268fd0 is 50, key is test_row_0/A:col10/1733708459979/Put/seqid=0 2024-12-09T01:41:00,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:00,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:00,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741852_1028 (size=12001) 2024-12-09T01:41:00,346 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/8006b6bbf9904dd18d58061214268fd0 2024-12-09T01:41:00,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/18f41ffbf74b471783f071746d683b9b is 50, key is test_row_0/B:col10/1733708459979/Put/seqid=0 2024-12-09T01:41:00,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708520385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,401 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708520385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708520390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,403 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,404 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708520391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,411 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708520400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,413 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741853_1029 (size=12001) 2024-12-09T01:41:00,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708520505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708520514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708520515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,530 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708520524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708520515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,535 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,536 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708520714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,718 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708520735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708520735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708520744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708520744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:00,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:00,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:00,829 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/18f41ffbf74b471783f071746d683b9b 2024-12-09T01:41:00,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/763fbaa0f3f84e23b621ffa2c7971c74 is 50, key is test_row_0/C:col10/1733708459979/Put/seqid=0 2024-12-09T01:41:00,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T01:41:00,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741854_1030 (size=12001) 2024-12-09T01:41:00,921 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/763fbaa0f3f84e23b621ffa2c7971c74 2024-12-09T01:41:00,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/8006b6bbf9904dd18d58061214268fd0 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/8006b6bbf9904dd18d58061214268fd0 2024-12-09T01:41:00,949 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/8006b6bbf9904dd18d58061214268fd0, entries=150, sequenceid=110, filesize=11.7 K 2024-12-09T01:41:00,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/18f41ffbf74b471783f071746d683b9b as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/18f41ffbf74b471783f071746d683b9b 2024-12-09T01:41:00,969 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/18f41ffbf74b471783f071746d683b9b, entries=150, sequenceid=110, filesize=11.7 K 2024-12-09T01:41:00,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/763fbaa0f3f84e23b621ffa2c7971c74 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/763fbaa0f3f84e23b621ffa2c7971c74 2024-12-09T01:41:00,988 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/763fbaa0f3f84e23b621ffa2c7971c74, entries=150, sequenceid=110, filesize=11.7 K 2024-12-09T01:41:00,993 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 88980a4de80c5830889346818652c3a7 in 712ms, sequenceid=110, compaction requested=true 2024-12-09T01:41:00,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:00,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:00,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-09T01:41:00,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-09T01:41:01,004 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-09T01:41:01,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2330 sec 2024-12-09T01:41:01,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.2650 sec 2024-12-09T01:41:01,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T01:41:01,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:01,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:01,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:01,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:01,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:01,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:01,066 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/3cce67efe6ab418083eb6fb80635a474 is 50, key is test_row_0/A:col10/1733708460375/Put/seqid=0 2024-12-09T01:41:01,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708521072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,081 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708521072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708521075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,084 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708521077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708521080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741855_1031 (size=14391) 2024-12-09T01:41:01,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708521184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708521195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708521195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708521195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708521195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708521405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,417 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,419 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708521405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,420 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708521406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708521406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708521405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,425 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/3cce67efe6ab418083eb6fb80635a474 2024-12-09T01:41:01,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/50711638324049119da233c47265f07a is 50, key is test_row_0/B:col10/1733708460375/Put/seqid=0 2024-12-09T01:41:01,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741856_1032 (size=12051) 2024-12-09T01:41:01,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/50711638324049119da233c47265f07a 2024-12-09T01:41:01,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/12825a21d8624e6899044aa5bf1cafb2 is 50, key is test_row_0/C:col10/1733708460375/Put/seqid=0 2024-12-09T01:41:01,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741857_1033 (size=12051) 2024-12-09T01:41:01,656 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/12825a21d8624e6899044aa5bf1cafb2 2024-12-09T01:41:01,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/3cce67efe6ab418083eb6fb80635a474 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/3cce67efe6ab418083eb6fb80635a474 2024-12-09T01:41:01,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/3cce67efe6ab418083eb6fb80635a474, entries=200, sequenceid=130, filesize=14.1 K 2024-12-09T01:41:01,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/50711638324049119da233c47265f07a as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/50711638324049119da233c47265f07a 2024-12-09T01:41:01,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/50711638324049119da233c47265f07a, entries=150, sequenceid=130, filesize=11.8 K 2024-12-09T01:41:01,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/12825a21d8624e6899044aa5bf1cafb2 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/12825a21d8624e6899044aa5bf1cafb2 2024-12-09T01:41:01,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/12825a21d8624e6899044aa5bf1cafb2, entries=150, sequenceid=130, filesize=11.8 K 2024-12-09T01:41:01,729 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 88980a4de80c5830889346818652c3a7 in 689ms, sequenceid=130, compaction requested=true 2024-12-09T01:41:01,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:01,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:41:01,730 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:41:01,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:01,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:41:01,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:01,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:41:01,731 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:41:01,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:01,733 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55177 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:41:01,733 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/A is initiating minor compaction (all files) 2024-12-09T01:41:01,733 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/A in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:01,734 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ef564974680248f0a429d59e0d430cba, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/9e344e5757c2476fbc617b851fabc862, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/8006b6bbf9904dd18d58061214268fd0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/3cce67efe6ab418083eb6fb80635a474] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=53.9 K 2024-12-09T01:41:01,735 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48157 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:41:01,735 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/B is initiating minor compaction (all files) 2024-12-09T01:41:01,735 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/B in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:01,736 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/dc13c71485f4497c9f6c28a184dc7efe, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/7c94c8d74a334bdb85821b5311af04c1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/18f41ffbf74b471783f071746d683b9b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/50711638324049119da233c47265f07a] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=47.0 K 2024-12-09T01:41:01,736 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef564974680248f0a429d59e0d430cba, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733708459488 2024-12-09T01:41:01,737 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting dc13c71485f4497c9f6c28a184dc7efe, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733708459488 2024-12-09T01:41:01,737 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c94c8d74a334bdb85821b5311af04c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733708459900 2024-12-09T01:41:01,738 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e344e5757c2476fbc617b851fabc862, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733708459811 2024-12-09T01:41:01,739 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 18f41ffbf74b471783f071746d683b9b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733708459971 2024-12-09T01:41:01,740 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8006b6bbf9904dd18d58061214268fd0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733708459971 2024-12-09T01:41:01,740 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 50711638324049119da233c47265f07a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733708460375 2024-12-09T01:41:01,741 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cce67efe6ab418083eb6fb80635a474, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733708460375 2024-12-09T01:41:01,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T01:41:01,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:01,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:01,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:01,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:01,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:01,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:01,754 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:01,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/89f7b99f4bd8402687bf64cfaa87e6a3 is 50, key is test_row_0/A:col10/1733708461073/Put/seqid=0 2024-12-09T01:41:01,780 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#A#compaction#22 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:01,781 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/7b4c717295f6420f9466acf91028d833 is 50, key is test_row_0/A:col10/1733708460375/Put/seqid=0 2024-12-09T01:41:01,789 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#B#compaction#23 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:01,794 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/a35e6eb36fe34285a91ead571082ee95 is 50, key is test_row_0/B:col10/1733708460375/Put/seqid=0 2024-12-09T01:41:01,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708521800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708521807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708521809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708521813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708521816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,826 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741858_1034 (size=12151) 2024-12-09T01:41:01,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/89f7b99f4bd8402687bf64cfaa87e6a3 2024-12-09T01:41:01,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741859_1035 (size=12291) 2024-12-09T01:41:01,869 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/7b4c717295f6420f9466acf91028d833 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/7b4c717295f6420f9466acf91028d833 2024-12-09T01:41:01,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741860_1036 (size=12291) 2024-12-09T01:41:01,890 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88980a4de80c5830889346818652c3a7/A of 88980a4de80c5830889346818652c3a7 into 7b4c717295f6420f9466acf91028d833(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:01,890 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:01,890 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/A, priority=12, startTime=1733708461730; duration=0sec 2024-12-09T01:41:01,890 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:01,890 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:A 2024-12-09T01:41:01,890 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:41:01,894 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/a35e6eb36fe34285a91ead571082ee95 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/a35e6eb36fe34285a91ead571082ee95 2024-12-09T01:41:01,896 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48157 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:41:01,896 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/C is initiating minor compaction (all files) 2024-12-09T01:41:01,896 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/C in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:01,897 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/5cb06c2dcb524d1995af449df58877bd, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/48a6ef2782574928845a2761d501354e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/763fbaa0f3f84e23b621ffa2c7971c74, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/12825a21d8624e6899044aa5bf1cafb2] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=47.0 K 2024-12-09T01:41:01,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/eaa6d826128147f6a06b86579d748164 is 50, key is test_row_0/B:col10/1733708461073/Put/seqid=0 2024-12-09T01:41:01,907 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cb06c2dcb524d1995af449df58877bd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733708459488 2024-12-09T01:41:01,907 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48a6ef2782574928845a2761d501354e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1733708459900 2024-12-09T01:41:01,909 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 763fbaa0f3f84e23b621ffa2c7971c74, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1733708459971 2024-12-09T01:41:01,909 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12825a21d8624e6899044aa5bf1cafb2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733708460375 2024-12-09T01:41:01,915 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88980a4de80c5830889346818652c3a7/B of 88980a4de80c5830889346818652c3a7 into a35e6eb36fe34285a91ead571082ee95(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:01,915 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:01,915 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/B, priority=12, startTime=1733708461730; duration=0sec 2024-12-09T01:41:01,915 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:01,916 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:B 2024-12-09T01:41:01,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708521924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708521924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,937 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,938 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708521924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,938 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708521934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,946 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:01,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708521934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:01,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:01,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:01,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741861_1037 (size=12151) 2024-12-09T01:41:01,954 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#C#compaction#25 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:01,955 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/22567e1ff4c743e9beb281fd3fdb4eb1 is 50, key is test_row_0/C:col10/1733708460375/Put/seqid=0 2024-12-09T01:41:01,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741862_1038 (size=12291) 2024-12-09T01:41:02,011 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/22567e1ff4c743e9beb281fd3fdb4eb1 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/22567e1ff4c743e9beb281fd3fdb4eb1 2024-12-09T01:41:02,035 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88980a4de80c5830889346818652c3a7/C of 88980a4de80c5830889346818652c3a7 into 22567e1ff4c743e9beb281fd3fdb4eb1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:02,035 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:02,036 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/C, priority=12, startTime=1733708461730; duration=0sec 2024-12-09T01:41:02,041 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:02,041 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:C 2024-12-09T01:41:02,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708522145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:02,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:02,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:02,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:02,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708522145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:02,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:02,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708522145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:02,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:02,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708522154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:02,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:02,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:02,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:02,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708522155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:02,167 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:02,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/eaa6d826128147f6a06b86579d748164 2024-12-09T01:41:02,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/62f6339c290b45afb2243d432a889f4a is 50, key is test_row_0/C:col10/1733708461073/Put/seqid=0 2024-12-09T01:41:02,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741863_1039 (size=12151) 2024-12-09T01:41:02,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:02,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708522454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:02,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:02,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:02,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:02,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:02,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708522454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:02,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:02,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:02,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708522454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:02,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:02,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:02,467 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:02,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:02,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708522474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:02,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:02,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708522474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:02,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:02,482 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:02,483 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:02,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:02,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:02,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:02,629 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T01:41:02,731 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T01:41:02,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/62f6339c290b45afb2243d432a889f4a 2024-12-09T01:41:02,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/89f7b99f4bd8402687bf64cfaa87e6a3 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/89f7b99f4bd8402687bf64cfaa87e6a3 2024-12-09T01:41:02,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/89f7b99f4bd8402687bf64cfaa87e6a3, entries=150, sequenceid=144, filesize=11.9 K 2024-12-09T01:41:02,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/eaa6d826128147f6a06b86579d748164 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/eaa6d826128147f6a06b86579d748164 2024-12-09T01:41:02,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/eaa6d826128147f6a06b86579d748164, entries=150, sequenceid=144, filesize=11.9 K 2024-12-09T01:41:02,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/62f6339c290b45afb2243d432a889f4a as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/62f6339c290b45afb2243d432a889f4a 2024-12-09T01:41:02,908 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/62f6339c290b45afb2243d432a889f4a, entries=150, sequenceid=144, filesize=11.9 K 2024-12-09T01:41:02,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 88980a4de80c5830889346818652c3a7 in 1177ms, sequenceid=144, compaction requested=false 2024-12-09T01:41:02,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:02,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-12-09T01:41:02,935 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:41:02,935 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:41:02,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-12-09T01:41:02,940 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:41:02,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T01:41:02,942 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:41:02,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:41:02,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:02,970 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T01:41:02,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:02,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:02,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:02,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:02,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:02,970 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:02,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/f8d951a721c645e7ba0f4a36af10fe1a is 50, key is test_row_0/A:col10/1733708461812/Put/seqid=0 2024-12-09T01:41:03,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:03,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708523007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:03,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:03,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:03,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708523010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:03,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:03,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708523015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:03,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:03,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708523017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,028 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:03,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:03,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708523019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:03,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:03,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T01:41:03,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741864_1040 (size=12151) 2024-12-09T01:41:03,100 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T01:41:03,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:03,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:03,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:03,101 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:03,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:03,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708523124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:03,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708523125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:03,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708523136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:03,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:03,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708523136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:03,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:03,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:03,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708523136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:03,139 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:03,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:03,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T01:41:03,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T01:41:03,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:03,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:03,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:03,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:03,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:03,348 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:03,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708523344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:03,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:03,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708523345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:03,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:03,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:03,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708523345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:03,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:03,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708523344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:03,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708523346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:03,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:03,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,356 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:03,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T01:41:03,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:03,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:03,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:03,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:03,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:03,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:03,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/f8d951a721c645e7ba0f4a36af10fe1a 2024-12-09T01:41:03,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/e6d3a714b08140fea39f9f5e659b10b8 is 50, key is test_row_0/B:col10/1733708461812/Put/seqid=0 2024-12-09T01:41:03,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741865_1041 (size=12151) 2024-12-09T01:41:03,554 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T01:41:03,554 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-09T01:41:03,557 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T01:41:03,557 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-09T01:41:03,558 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-09T01:41:03,558 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-09T01:41:03,559 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-12-09T01:41:03,559 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-12-09T01:41:03,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T01:41:03,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T01:41:03,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:03,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:03,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:03,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:03,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:03,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:03,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:03,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:03,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708523664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:03,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708523664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:03,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:03,675 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:03,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708523665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:03,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708523665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:03,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708523665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:03,676 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:03,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:03,677 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:03,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T01:41:03,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:03,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:03,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:03,743 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:03,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:03,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:03,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T01:41:03,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:03,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:03,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:03,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:03,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:03,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:03,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/e6d3a714b08140fea39f9f5e659b10b8 2024-12-09T01:41:03,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/91332d98587d45b28851d11b5bd3ea16 is 50, key is test_row_0/C:col10/1733708461812/Put/seqid=0 2024-12-09T01:41:03,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741866_1042 (size=12151) 2024-12-09T01:41:04,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T01:41:04,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:04,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:04,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:04,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:04,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:04,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:04,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T01:41:04,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708524184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:04,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708524184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:04,187 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:04,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:04,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708524184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:04,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:04,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:04,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:04,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708524184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:04,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:04,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:04,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:04,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708524185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:04,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:04,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:04,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:04,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:04,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T01:41:04,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:04,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:04,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:04,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:04,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:04,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:04,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/91332d98587d45b28851d11b5bd3ea16 2024-12-09T01:41:04,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T01:41:04,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:04,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:04,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:04,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:04,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:04,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:04,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/f8d951a721c645e7ba0f4a36af10fe1a as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/f8d951a721c645e7ba0f4a36af10fe1a 2024-12-09T01:41:04,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/f8d951a721c645e7ba0f4a36af10fe1a, entries=150, sequenceid=166, filesize=11.9 K 2024-12-09T01:41:04,400 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/e6d3a714b08140fea39f9f5e659b10b8 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/e6d3a714b08140fea39f9f5e659b10b8 2024-12-09T01:41:04,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/e6d3a714b08140fea39f9f5e659b10b8, entries=150, sequenceid=166, filesize=11.9 K 2024-12-09T01:41:04,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/91332d98587d45b28851d11b5bd3ea16 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/91332d98587d45b28851d11b5bd3ea16 2024-12-09T01:41:04,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/91332d98587d45b28851d11b5bd3ea16, entries=150, sequenceid=166, filesize=11.9 K 2024-12-09T01:41:04,433 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 88980a4de80c5830889346818652c3a7 in 1463ms, sequenceid=166, compaction requested=true 2024-12-09T01:41:04,433 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:04,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:41:04,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:04,433 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:04,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:41:04,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:04,433 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:04,435 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36593 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:04,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:41:04,435 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/A is initiating minor compaction (all files) 2024-12-09T01:41:04,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:04,435 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/A in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:04,436 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/7b4c717295f6420f9466acf91028d833, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/89f7b99f4bd8402687bf64cfaa87e6a3, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/f8d951a721c645e7ba0f4a36af10fe1a] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=35.7 K 2024-12-09T01:41:04,436 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36593 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:04,436 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/B is initiating minor compaction (all files) 2024-12-09T01:41:04,436 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/B in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:04,437 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/a35e6eb36fe34285a91ead571082ee95, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/eaa6d826128147f6a06b86579d748164, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/e6d3a714b08140fea39f9f5e659b10b8] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=35.7 K 2024-12-09T01:41:04,440 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b4c717295f6420f9466acf91028d833, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733708460375 2024-12-09T01:41:04,440 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting a35e6eb36fe34285a91ead571082ee95, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733708460375 2024-12-09T01:41:04,441 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89f7b99f4bd8402687bf64cfaa87e6a3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733708461073 2024-12-09T01:41:04,442 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting eaa6d826128147f6a06b86579d748164, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733708461073 2024-12-09T01:41:04,442 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8d951a721c645e7ba0f4a36af10fe1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733708461781 2024-12-09T01:41:04,443 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting e6d3a714b08140fea39f9f5e659b10b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733708461781 2024-12-09T01:41:04,471 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#B#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:04,472 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/b240cc493ea74527859194bbb08c89f9 is 50, key is test_row_0/B:col10/1733708461812/Put/seqid=0 2024-12-09T01:41:04,475 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#A#compaction#31 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:04,475 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/45fe309f77774ceebe76fb01f4dc554f is 50, key is test_row_0/A:col10/1733708461812/Put/seqid=0 2024-12-09T01:41:04,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741867_1043 (size=12493) 2024-12-09T01:41:04,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741868_1044 (size=12493) 2024-12-09T01:41:04,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-12-09T01:41:04,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:04,531 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T01:41:04,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:04,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:04,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:04,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:04,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:04,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:04,533 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/45fe309f77774ceebe76fb01f4dc554f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/45fe309f77774ceebe76fb01f4dc554f 2024-12-09T01:41:04,545 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/A of 88980a4de80c5830889346818652c3a7 into 45fe309f77774ceebe76fb01f4dc554f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:04,545 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:04,545 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/A, priority=13, startTime=1733708464433; duration=0sec 2024-12-09T01:41:04,545 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:04,545 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:A 2024-12-09T01:41:04,545 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:04,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/df8ae9e610b249cfbc5e12dae8635f0e is 50, key is test_row_0/A:col10/1733708463015/Put/seqid=0 2024-12-09T01:41:04,555 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36593 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:04,556 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/C is initiating minor compaction (all files) 2024-12-09T01:41:04,556 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/C in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:04,556 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/22567e1ff4c743e9beb281fd3fdb4eb1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/62f6339c290b45afb2243d432a889f4a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/91332d98587d45b28851d11b5bd3ea16] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=35.7 K 2024-12-09T01:41:04,557 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22567e1ff4c743e9beb281fd3fdb4eb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733708460375 2024-12-09T01:41:04,557 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62f6339c290b45afb2243d432a889f4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1733708461073 2024-12-09T01:41:04,558 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91332d98587d45b28851d11b5bd3ea16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733708461781 2024-12-09T01:41:04,587 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#C#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:04,588 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/0f4c51d39dd344d0936abb6e8475535a is 50, key is test_row_0/C:col10/1733708461812/Put/seqid=0 2024-12-09T01:41:04,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741869_1045 (size=9757) 2024-12-09T01:41:04,601 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/df8ae9e610b249cfbc5e12dae8635f0e 2024-12-09T01:41:04,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741870_1046 (size=12493) 2024-12-09T01:41:04,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/24589f63001e46099e89f7fbb3a96f67 is 50, key is test_row_0/B:col10/1733708463015/Put/seqid=0 2024-12-09T01:41:04,655 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/0f4c51d39dd344d0936abb6e8475535a as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/0f4c51d39dd344d0936abb6e8475535a 2024-12-09T01:41:04,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741871_1047 (size=9757) 2024-12-09T01:41:04,669 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/C of 88980a4de80c5830889346818652c3a7 into 0f4c51d39dd344d0936abb6e8475535a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:04,669 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:04,670 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/C, priority=13, startTime=1733708464435; duration=0sec 2024-12-09T01:41:04,670 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:04,670 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:C 2024-12-09T01:41:04,671 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/24589f63001e46099e89f7fbb3a96f67 2024-12-09T01:41:04,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/c7ef9518edd04457a1bb158b9cc4ad0e is 50, key is test_row_0/C:col10/1733708463015/Put/seqid=0 2024-12-09T01:41:04,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741872_1048 (size=9757) 2024-12-09T01:41:04,925 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/b240cc493ea74527859194bbb08c89f9 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/b240cc493ea74527859194bbb08c89f9 2024-12-09T01:41:04,937 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/B of 88980a4de80c5830889346818652c3a7 into b240cc493ea74527859194bbb08c89f9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:04,937 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:04,937 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/B, priority=13, startTime=1733708464433; duration=0sec 2024-12-09T01:41:04,937 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:04,937 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:B 2024-12-09T01:41:05,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T01:41:05,103 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/c7ef9518edd04457a1bb158b9cc4ad0e 2024-12-09T01:41:05,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/df8ae9e610b249cfbc5e12dae8635f0e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/df8ae9e610b249cfbc5e12dae8635f0e 2024-12-09T01:41:05,126 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/df8ae9e610b249cfbc5e12dae8635f0e, entries=100, sequenceid=179, filesize=9.5 K 2024-12-09T01:41:05,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/24589f63001e46099e89f7fbb3a96f67 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/24589f63001e46099e89f7fbb3a96f67 2024-12-09T01:41:05,139 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/24589f63001e46099e89f7fbb3a96f67, entries=100, sequenceid=179, filesize=9.5 K 2024-12-09T01:41:05,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/c7ef9518edd04457a1bb158b9cc4ad0e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/c7ef9518edd04457a1bb158b9cc4ad0e 2024-12-09T01:41:05,160 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/c7ef9518edd04457a1bb158b9cc4ad0e, entries=100, sequenceid=179, filesize=9.5 K 2024-12-09T01:41:05,163 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 88980a4de80c5830889346818652c3a7 in 632ms, sequenceid=179, compaction requested=false 2024-12-09T01:41:05,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:05,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:05,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-09T01:41:05,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-09T01:41:05,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-09T01:41:05,179 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2330 sec 2024-12-09T01:41:05,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.2450 sec 2024-12-09T01:41:05,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:41:05,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:05,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:05,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:05,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:05,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:05,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:05,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/01da971ef2024d15b090f5f6143e576a is 50, key is test_row_0/A:col10/1733708465209/Put/seqid=0 2024-12-09T01:41:05,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741873_1049 (size=14541) 2024-12-09T01:41:05,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/01da971ef2024d15b090f5f6143e576a 2024-12-09T01:41:05,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708465263, max timestamp=1733708465278Num uniques 859; 2024-12-09T01:41:05,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:41:05,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708465263, max timestamp=1733708465278Num uniques 859; 2024-12-09T01:41:05,298 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:41:05,314 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733708465263, max timestamp=1733708465286Num uniques 859; 2024-12-09T01:41:05,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T01:41:05,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/ae2c8be98d0f4536bc18e6c10a69bacb is 50, key is test_row_0/B:col10/1733708465209/Put/seqid=0 2024-12-09T01:41:05,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:05,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708525335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:05,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708525336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:05,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:05,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:05,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708525341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:05,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,357 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:05,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:05,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708525343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:05,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:05,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708525349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:05,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:05,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:05,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741874_1050 (size=12151) 2024-12-09T01:41:05,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/ae2c8be98d0f4536bc18e6c10a69bacb 2024-12-09T01:41:05,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/5f55ffbac1fb49149840b4380b5506b6 is 50, key is test_row_0/C:col10/1733708465209/Put/seqid=0 2024-12-09T01:41:05,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741875_1051 (size=12151) 2024-12-09T01:41:05,435 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/5f55ffbac1fb49149840b4380b5506b6 2024-12-09T01:41:05,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/01da971ef2024d15b090f5f6143e576a as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/01da971ef2024d15b090f5f6143e576a 2024-12-09T01:41:05,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/01da971ef2024d15b090f5f6143e576a, entries=200, sequenceid=192, filesize=14.2 K 2024-12-09T01:41:05,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/ae2c8be98d0f4536bc18e6c10a69bacb as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/ae2c8be98d0f4536bc18e6c10a69bacb 2024-12-09T01:41:05,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708525465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:05,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:05,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:05,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708525465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708525466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:05,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:05,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708525467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:05,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:05,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:05,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708525467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:05,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:05,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/ae2c8be98d0f4536bc18e6c10a69bacb, entries=150, sequenceid=192, filesize=11.9 K 2024-12-09T01:41:05,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/5f55ffbac1fb49149840b4380b5506b6 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/5f55ffbac1fb49149840b4380b5506b6 2024-12-09T01:41:05,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/5f55ffbac1fb49149840b4380b5506b6, entries=150, sequenceid=192, filesize=11.9 K 2024-12-09T01:41:05,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=133.78 KB/136995 for 88980a4de80c5830889346818652c3a7 in 283ms, sequenceid=192, compaction requested=true 2024-12-09T01:41:05,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:05,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:41:05,502 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:05,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:05,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:41:05,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:05,502 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:05,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:41:05,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:05,504 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36791 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:05,505 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/A is initiating minor compaction (all files) 2024-12-09T01:41:05,505 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/A in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:05,505 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/45fe309f77774ceebe76fb01f4dc554f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/df8ae9e610b249cfbc5e12dae8635f0e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/01da971ef2024d15b090f5f6143e576a] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=35.9 K 2024-12-09T01:41:05,508 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45fe309f77774ceebe76fb01f4dc554f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733708461781 2024-12-09T01:41:05,508 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:05,508 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/B is initiating minor compaction (all files) 2024-12-09T01:41:05,508 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/B in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:05,508 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/b240cc493ea74527859194bbb08c89f9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/24589f63001e46099e89f7fbb3a96f67, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/ae2c8be98d0f4536bc18e6c10a69bacb] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=33.6 K 2024-12-09T01:41:05,509 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting df8ae9e610b249cfbc5e12dae8635f0e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733708463012 2024-12-09T01:41:05,509 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting b240cc493ea74527859194bbb08c89f9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733708461781 2024-12-09T01:41:05,510 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 24589f63001e46099e89f7fbb3a96f67, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733708463012 2024-12-09T01:41:05,510 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01da971ef2024d15b090f5f6143e576a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733708465203 2024-12-09T01:41:05,511 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting ae2c8be98d0f4536bc18e6c10a69bacb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733708465203 2024-12-09T01:41:05,530 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#A#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:05,531 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/ac30044b395a49068753ed0fc592ca0f is 50, key is test_row_0/A:col10/1733708465209/Put/seqid=0 2024-12-09T01:41:05,548 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#B#compaction#40 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:05,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741876_1052 (size=12595) 2024-12-09T01:41:05,549 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/bb0bf59b992b4bcea88e8dfb964d195d is 50, key is test_row_0/B:col10/1733708465209/Put/seqid=0 2024-12-09T01:41:05,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741877_1053 (size=12595) 2024-12-09T01:41:05,587 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/bb0bf59b992b4bcea88e8dfb964d195d as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/bb0bf59b992b4bcea88e8dfb964d195d 2024-12-09T01:41:05,596 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/B of 88980a4de80c5830889346818652c3a7 into bb0bf59b992b4bcea88e8dfb964d195d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:05,596 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:05,596 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/B, priority=13, startTime=1733708465502; duration=0sec 2024-12-09T01:41:05,596 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:05,597 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:B 2024-12-09T01:41:05,597 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:05,601 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:05,601 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/C is initiating minor compaction (all files) 2024-12-09T01:41:05,601 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/C in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:05,601 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/0f4c51d39dd344d0936abb6e8475535a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/c7ef9518edd04457a1bb158b9cc4ad0e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/5f55ffbac1fb49149840b4380b5506b6] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=33.6 K 2024-12-09T01:41:05,602 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f4c51d39dd344d0936abb6e8475535a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1733708461781 2024-12-09T01:41:05,602 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting c7ef9518edd04457a1bb158b9cc4ad0e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1733708463012 2024-12-09T01:41:05,603 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f55ffbac1fb49149840b4380b5506b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733708465203 2024-12-09T01:41:05,620 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#C#compaction#41 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:05,621 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/055f71331c2842aba3f44576f91439a5 is 50, key is test_row_0/C:col10/1733708465209/Put/seqid=0 2024-12-09T01:41:05,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741878_1054 (size=12595) 2024-12-09T01:41:05,660 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/055f71331c2842aba3f44576f91439a5 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/055f71331c2842aba3f44576f91439a5 2024-12-09T01:41:05,676 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/C of 88980a4de80c5830889346818652c3a7 into 055f71331c2842aba3f44576f91439a5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:05,676 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:05,676 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/C, priority=13, startTime=1733708465502; duration=0sec 2024-12-09T01:41:05,676 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:05,676 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:C 2024-12-09T01:41:05,685 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733708465332, max timestamp=1733708465679Num uniques 1018; 2024-12-09T01:41:05,686 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T01:41:05,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:05,688 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=109.18 KB heapSize=311.21 KB 2024-12-09T01:41:05,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:05,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T01:41:05,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:05,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:05,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:05,690 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T01:41:05,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/3554977c95d24546806cae9740a46f17 is 50, key is test_row_0/A:col10/1733708465338/Put/seqid=0 2024-12-09T01:41:05,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:05,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708525717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:05,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708525719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708525719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:05,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:05,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:05,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708525722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,733 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:05,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708525725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:05,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741879_1055 (size=16931) 2024-12-09T01:41:05,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:05,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708525834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:05,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:05,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:05,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708525834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:05,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,842 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:05,846 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:05,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708525843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:05,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:05,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:05,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708525843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:05,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,848 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:05,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:05,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708525844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:05,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:05,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:05,957 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/ac30044b395a49068753ed0fc592ca0f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ac30044b395a49068753ed0fc592ca0f 2024-12-09T01:41:05,969 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/A of 88980a4de80c5830889346818652c3a7 into ac30044b395a49068753ed0fc592ca0f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:05,969 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:05,969 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/A, priority=13, startTime=1733708465502; duration=0sec 2024-12-09T01:41:05,969 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:05,969 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:A 2024-12-09T01:41:06,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:06,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708526053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:06,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708526054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:06,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:06,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:06,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708526058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:06,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:06,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:06,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708526058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:06,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:06,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:06,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708526058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:06,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:06,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/3554977c95d24546806cae9740a46f17 2024-12-09T01:41:06,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/713b6efd6e0a4d04ae2017115d23e426 is 50, key is test_row_0/B:col10/1733708465338/Put/seqid=0 2024-12-09T01:41:06,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741880_1056 (size=12151) 2024-12-09T01:41:06,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708526364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:06,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:06,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:06,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708526365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:06,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:06,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:06,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708526365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:06,368 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:06,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:06,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708526374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:06,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:06,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708526375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:06,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,381 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:06,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,382 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:06,603 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/713b6efd6e0a4d04ae2017115d23e426 2024-12-09T01:41:06,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/d665d3f5011b48fbacd94db86d7d50d6 is 50, key is test_row_0/C:col10/1733708465338/Put/seqid=0 2024-12-09T01:41:06,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741881_1057 (size=12151) 2024-12-09T01:41:06,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:06,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708526874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:06,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:06,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:06,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708526875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:06,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:06,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:06,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708526875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:06,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,877 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:06,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:06,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708526894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:06,897 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:06,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708526894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:06,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:07,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48.80 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/d665d3f5011b48fbacd94db86d7d50d6 2024-12-09T01:41:07,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/3554977c95d24546806cae9740a46f17 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/3554977c95d24546806cae9740a46f17 2024-12-09T01:41:07,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/3554977c95d24546806cae9740a46f17, entries=250, sequenceid=233, filesize=16.5 K 2024-12-09T01:41:07,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/713b6efd6e0a4d04ae2017115d23e426 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/713b6efd6e0a4d04ae2017115d23e426 2024-12-09T01:41:07,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/713b6efd6e0a4d04ae2017115d23e426, entries=150, sequenceid=233, filesize=11.9 K 2024-12-09T01:41:07,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/d665d3f5011b48fbacd94db86d7d50d6 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/d665d3f5011b48fbacd94db86d7d50d6 2024-12-09T01:41:07,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/d665d3f5011b48fbacd94db86d7d50d6, entries=150, sequenceid=233, filesize=11.9 K 2024-12-09T01:41:07,077 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~109.18 KB/111805, heapSize ~311.16 KB/318632, currentSize=73.80 KB/75570 for 88980a4de80c5830889346818652c3a7 in 1389ms, sequenceid=233, compaction requested=false 2024-12-09T01:41:07,078 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:07,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-09T01:41:07,096 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:41:07,097 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:41:07,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-12-09T01:41:07,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T01:41:07,102 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:41:07,104 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:41:07,105 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:41:07,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T01:41:07,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-12-09T01:41:07,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:07,258 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T01:41:07,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:07,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:07,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:07,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:07,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:07,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:07,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/700e2649495c43adad5b4248567d5ab1 is 50, key is test_row_0/A:col10/1733708465719/Put/seqid=0 2024-12-09T01:41:07,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741882_1058 (size=12151) 2024-12-09T01:41:07,275 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/700e2649495c43adad5b4248567d5ab1 2024-12-09T01:41:07,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/6ca47ba959db4f50bca5c39de4976c7d is 50, key is test_row_0/B:col10/1733708465719/Put/seqid=0 2024-12-09T01:41:07,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741883_1059 (size=12151) 2024-12-09T01:41:07,295 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/6ca47ba959db4f50bca5c39de4976c7d 2024-12-09T01:41:07,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/de56646fa92242cf90614c3d77bc5d89 is 50, key is test_row_0/C:col10/1733708465719/Put/seqid=0 2024-12-09T01:41:07,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741884_1060 (size=12151) 2024-12-09T01:41:07,319 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/de56646fa92242cf90614c3d77bc5d89 2024-12-09T01:41:07,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/700e2649495c43adad5b4248567d5ab1 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/700e2649495c43adad5b4248567d5ab1 2024-12-09T01:41:07,341 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/700e2649495c43adad5b4248567d5ab1, entries=150, sequenceid=248, filesize=11.9 K 2024-12-09T01:41:07,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/6ca47ba959db4f50bca5c39de4976c7d as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/6ca47ba959db4f50bca5c39de4976c7d 2024-12-09T01:41:07,353 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/6ca47ba959db4f50bca5c39de4976c7d, entries=150, sequenceid=248, filesize=11.9 K 2024-12-09T01:41:07,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/de56646fa92242cf90614c3d77bc5d89 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/de56646fa92242cf90614c3d77bc5d89 2024-12-09T01:41:07,362 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/de56646fa92242cf90614c3d77bc5d89, entries=150, sequenceid=248, filesize=11.9 K 2024-12-09T01:41:07,363 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for 88980a4de80c5830889346818652c3a7 in 105ms, sequenceid=248, compaction requested=true 2024-12-09T01:41:07,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:07,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:07,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-09T01:41:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-09T01:41:07,369 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-09T01:41:07,369 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 261 msec 2024-12-09T01:41:07,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 273 msec 2024-12-09T01:41:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-09T01:41:07,414 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:41:07,415 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:41:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-12-09T01:41:07,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T01:41:07,418 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:41:07,421 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:41:07,421 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:41:07,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T01:41:07,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-12-09T01:41:07,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:07,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:07,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:07,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-09T01:41:07,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-09T01:41:07,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-09T01:41:07,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 161 msec 2024-12-09T01:41:07,589 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 172 msec 2024-12-09T01:41:07,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-09T01:41:07,735 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:41:07,736 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:41:07,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-12-09T01:41:07,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T01:41:07,741 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:41:07,743 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:41:07,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:41:07,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T01:41:07,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-09T01:41:07,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:07,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:07,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:07,898 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:41:07,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:07,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:07,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:07,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:07,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:07,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:07,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:07,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:07,898 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:07,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:07,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/1022950a507d4fbe81a576bbe7a91bcd is 50, key is test_row_0/A:col10/1733708467896/Put/seqid=0 2024-12-09T01:41:07,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741885_1061 (size=14687) 2024-12-09T01:41:07,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/1022950a507d4fbe81a576bbe7a91bcd 2024-12-09T01:41:07,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:07,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708527936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:07,948 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708467928, max timestamp=1733708467940Num uniques 859; 2024-12-09T01:41:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:07,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:07,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:41:07,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708467928, max timestamp=1733708467940Num uniques 859; 2024-12-09T01:41:07,949 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:41:07,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708467928, max timestamp=1733708467940Num uniques 859; 2024-12-09T01:41:07,950 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:41:07,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/aec2b9f6ef1847d58020d79ef84275d7 is 50, key is test_row_0/B:col10/1733708467896/Put/seqid=0 2024-12-09T01:41:07,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741886_1062 (size=9857) 2024-12-09T01:41:07,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/aec2b9f6ef1847d58020d79ef84275d7 2024-12-09T01:41:07,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/ad1c1fbc4d2f4d339bdd7185352d07eb is 50, key is test_row_0/C:col10/1733708467896/Put/seqid=0 2024-12-09T01:41:08,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:08,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708527990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:08,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733708467969, max timestamp=1733708467996Num uniques 1018; 2024-12-09T01:41:08,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:08,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T01:41:08,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708527991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:08,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733708467969, max timestamp=1733708467996Num uniques 1018; 2024-12-09T01:41:08,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T01:41:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,002 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:08,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733708467969, max timestamp=1733708467996Num uniques 1018; 2024-12-09T01:41:08,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T01:41:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:08,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741887_1063 (size=9857) 2024-12-09T01:41:08,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/ad1c1fbc4d2f4d339bdd7185352d07eb 2024-12-09T01:41:08,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/1022950a507d4fbe81a576bbe7a91bcd as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/1022950a507d4fbe81a576bbe7a91bcd 2024-12-09T01:41:08,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/1022950a507d4fbe81a576bbe7a91bcd, entries=200, sequenceid=259, filesize=14.3 K 2024-12-09T01:41:08,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/aec2b9f6ef1847d58020d79ef84275d7 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/aec2b9f6ef1847d58020d79ef84275d7 2024-12-09T01:41:08,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/aec2b9f6ef1847d58020d79ef84275d7, entries=100, sequenceid=259, filesize=9.6 K 2024-12-09T01:41:08,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/ad1c1fbc4d2f4d339bdd7185352d07eb as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/ad1c1fbc4d2f4d339bdd7185352d07eb 2024-12-09T01:41:08,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/ad1c1fbc4d2f4d339bdd7185352d07eb, entries=100, sequenceid=259, filesize=9.6 K 2024-12-09T01:41:08,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=80.51 KB/82440 for 88980a4de80c5830889346818652c3a7 in 150ms, sequenceid=259, compaction requested=true 2024-12-09T01:41:08,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:08,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:41:08,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:08,048 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:41:08,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:41:08,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:08,048 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:41:08,048 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:41:08,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:08,050 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 56364 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:41:08,050 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/A is initiating minor compaction (all files) 2024-12-09T01:41:08,051 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/A in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:08,051 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ac30044b395a49068753ed0fc592ca0f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/3554977c95d24546806cae9740a46f17, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/700e2649495c43adad5b4248567d5ab1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/1022950a507d4fbe81a576bbe7a91bcd] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=55.0 K 2024-12-09T01:41:08,052 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac30044b395a49068753ed0fc592ca0f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733708465203 2024-12-09T01:41:08,052 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3554977c95d24546806cae9740a46f17, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733708465332 2024-12-09T01:41:08,053 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46754 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:41:08,053 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 700e2649495c43adad5b4248567d5ab1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733708465717 2024-12-09T01:41:08,053 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/B is initiating minor compaction (all files) 2024-12-09T01:41:08,053 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/B in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:08,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-12-09T01:41:08,053 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/bb0bf59b992b4bcea88e8dfb964d195d, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/713b6efd6e0a4d04ae2017115d23e426, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/6ca47ba959db4f50bca5c39de4976c7d, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/aec2b9f6ef1847d58020d79ef84275d7] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=45.7 K 2024-12-09T01:41:08,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:08,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:08,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T01:41:08,054 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=87.22 KB heapSize=240 KB 2024-12-09T01:41:08,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:08,054 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting bb0bf59b992b4bcea88e8dfb964d195d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733708465203 2024-12-09T01:41:08,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:08,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:08,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:08,055 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 713b6efd6e0a4d04ae2017115d23e426, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733708465332 2024-12-09T01:41:08,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:08,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:08,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:08,055 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1022950a507d4fbe81a576bbe7a91bcd, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733708467886 2024-12-09T01:41:08,056 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ca47ba959db4f50bca5c39de4976c7d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733708465717 2024-12-09T01:41:08,057 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting aec2b9f6ef1847d58020d79ef84275d7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733708467892 2024-12-09T01:41:08,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/a18c98172af7451b826ed629d0356c2e is 50, key is test_row_0/A:col10/1733708468050/Put/seqid=0 2024-12-09T01:41:08,090 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#B#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:08,091 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/61616af18917496b96cf1f059ebdf3a9 is 50, key is test_row_0/B:col10/1733708467896/Put/seqid=0 2024-12-09T01:41:08,101 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#A#compaction#53 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:08,102 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/9c7860baf5da452bb1d2293330403d79 is 50, key is test_row_0/A:col10/1733708467896/Put/seqid=0 2024-12-09T01:41:08,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:08,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708528111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:08,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:08,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708528114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:08,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:08,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708528115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:08,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,122 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:08,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:08,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708528115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:08,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:08,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:08,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708528118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:08,131 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,132 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:08,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741889_1065 (size=12831) 2024-12-09T01:41:08,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741890_1066 (size=12831) 2024-12-09T01:41:08,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741888_1064 (size=17181) 2024-12-09T01:41:08,153 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/9c7860baf5da452bb1d2293330403d79 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/9c7860baf5da452bb1d2293330403d79 2024-12-09T01:41:08,153 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/a18c98172af7451b826ed629d0356c2e 2024-12-09T01:41:08,169 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88980a4de80c5830889346818652c3a7/A of 88980a4de80c5830889346818652c3a7 into 9c7860baf5da452bb1d2293330403d79(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:08,169 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:08,169 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/A, priority=12, startTime=1733708468048; duration=0sec 2024-12-09T01:41:08,169 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:08,169 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:A 2024-12-09T01:41:08,169 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:41:08,171 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46754 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:41:08,171 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/C is initiating minor compaction (all files) 2024-12-09T01:41:08,171 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/C in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:08,171 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/055f71331c2842aba3f44576f91439a5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/d665d3f5011b48fbacd94db86d7d50d6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/de56646fa92242cf90614c3d77bc5d89, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/ad1c1fbc4d2f4d339bdd7185352d07eb] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=45.7 K 2024-12-09T01:41:08,172 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 055f71331c2842aba3f44576f91439a5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733708465203 2024-12-09T01:41:08,173 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting d665d3f5011b48fbacd94db86d7d50d6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1733708465332 2024-12-09T01:41:08,174 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting de56646fa92242cf90614c3d77bc5d89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1733708465717 2024-12-09T01:41:08,174 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad1c1fbc4d2f4d339bdd7185352d07eb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733708467892 2024-12-09T01:41:08,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/e61ec1ad015e457fa483a20f280db760 is 50, key is test_row_0/B:col10/1733708468050/Put/seqid=0 2024-12-09T01:41:08,205 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#C#compaction#55 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:08,206 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/3b574b2b58e84ae494e04067ece9c1f1 is 50, key is test_row_0/C:col10/1733708467896/Put/seqid=0 2024-12-09T01:41:08,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741891_1067 (size=12301) 2024-12-09T01:41:08,217 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/e61ec1ad015e457fa483a20f280db760 2024-12-09T01:41:08,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/2165e34413384c51a4d3eb5b3d9cf370 is 50, key is test_row_0/C:col10/1733708468050/Put/seqid=0 2024-12-09T01:41:08,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741892_1068 (size=12831) 2024-12-09T01:41:08,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:08,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708528233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:08,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:08,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708528224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:08,239 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:08,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741893_1069 (size=12301) 2024-12-09T01:41:08,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:08,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708528244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:08,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:08,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708528334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:08,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:08,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:08,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708528335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:08,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:08,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T01:41:08,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:08,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708528444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:08,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:08,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:08,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708528444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:08,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:08,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:08,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708528454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:08,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:08,545 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/61616af18917496b96cf1f059ebdf3a9 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/61616af18917496b96cf1f059ebdf3a9 2024-12-09T01:41:08,556 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88980a4de80c5830889346818652c3a7/B of 88980a4de80c5830889346818652c3a7 into 61616af18917496b96cf1f059ebdf3a9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:08,557 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:08,557 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/B, priority=12, startTime=1733708468048; duration=0sec 2024-12-09T01:41:08,557 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:08,557 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:B 2024-12-09T01:41:08,643 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/3b574b2b58e84ae494e04067ece9c1f1 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/3b574b2b58e84ae494e04067ece9c1f1 2024-12-09T01:41:08,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:08,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708528644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:08,647 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/2165e34413384c51a4d3eb5b3d9cf370 2024-12-09T01:41:08,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:08,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708528646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:08,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/a18c98172af7451b826ed629d0356c2e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/a18c98172af7451b826ed629d0356c2e 2024-12-09T01:41:08,659 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88980a4de80c5830889346818652c3a7/C of 88980a4de80c5830889346818652c3a7 into 3b574b2b58e84ae494e04067ece9c1f1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:08,659 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:08,659 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/C, priority=12, startTime=1733708468048; duration=0sec 2024-12-09T01:41:08,659 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:08,659 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:C 2024-12-09T01:41:08,662 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/a18c98172af7451b826ed629d0356c2e, entries=250, sequenceid=306, filesize=16.8 K 2024-12-09T01:41:08,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/e61ec1ad015e457fa483a20f280db760 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/e61ec1ad015e457fa483a20f280db760 2024-12-09T01:41:08,675 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/e61ec1ad015e457fa483a20f280db760, entries=150, sequenceid=306, filesize=12.0 K 2024-12-09T01:41:08,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/2165e34413384c51a4d3eb5b3d9cf370 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/2165e34413384c51a4d3eb5b3d9cf370 2024-12-09T01:41:08,686 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/2165e34413384c51a4d3eb5b3d9cf370, entries=150, sequenceid=306, filesize=12.0 K 2024-12-09T01:41:08,687 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~259.88 KB/266112, currentSize=87.22 KB/89310 for 88980a4de80c5830889346818652c3a7 in 633ms, sequenceid=306, compaction requested=false 2024-12-09T01:41:08,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:08,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:08,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-09T01:41:08,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-09T01:41:08,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-09T01:41:08,692 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 945 msec 2024-12-09T01:41:08,696 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 957 msec 2024-12-09T01:41:08,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T01:41:08,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:08,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:08,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:08,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:08,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:08,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:08,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:08,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/43fb489db2064920b061cc9615bae672 is 50, key is test_row_0/A:col10/1733708468098/Put/seqid=0 2024-12-09T01:41:08,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:08,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708528793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:08,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708528796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:08,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708528796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:08,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:08,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:08,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741894_1070 (size=17181) 2024-12-09T01:41:08,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/43fb489db2064920b061cc9615bae672 2024-12-09T01:41:08,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/fb376b7847244b068726dba10f721072 is 50, key is test_row_0/B:col10/1733708468098/Put/seqid=0 2024-12-09T01:41:08,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741895_1071 (size=12301) 2024-12-09T01:41:08,837 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/fb376b7847244b068726dba10f721072 2024-12-09T01:41:08,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/c4da6552ec6d45979faceedb34848577 is 50, key is test_row_0/C:col10/1733708468098/Put/seqid=0 2024-12-09T01:41:08,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741896_1072 (size=12301) 2024-12-09T01:41:08,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/c4da6552ec6d45979faceedb34848577 2024-12-09T01:41:08,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-09T01:41:08,873 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:41:08,874 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:41:08,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-12-09T01:41:08,878 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:41:08,879 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:41:08,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:41:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T01:41:08,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/43fb489db2064920b061cc9615bae672 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/43fb489db2064920b061cc9615bae672 2024-12-09T01:41:08,896 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/43fb489db2064920b061cc9615bae672, entries=250, sequenceid=327, filesize=16.8 K 2024-12-09T01:41:08,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/fb376b7847244b068726dba10f721072 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/fb376b7847244b068726dba10f721072 2024-12-09T01:41:08,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/fb376b7847244b068726dba10f721072, entries=150, sequenceid=327, filesize=12.0 K 2024-12-09T01:41:08,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/c4da6552ec6d45979faceedb34848577 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/c4da6552ec6d45979faceedb34848577 2024-12-09T01:41:08,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:08,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708528904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:08,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:08,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708528914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:08,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:08,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:08,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/c4da6552ec6d45979faceedb34848577, entries=150, sequenceid=327, filesize=12.0 K 2024-12-09T01:41:08,921 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 88980a4de80c5830889346818652c3a7 in 161ms, sequenceid=327, compaction requested=true 2024-12-09T01:41:08,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:08,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:41:08,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:08,922 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:08,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:41:08,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:08,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:41:08,922 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:08,922 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:08,923 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47193 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:08,923 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/A is initiating minor compaction (all files) 2024-12-09T01:41:08,923 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/A in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:08,923 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/9c7860baf5da452bb1d2293330403d79, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/a18c98172af7451b826ed629d0356c2e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/43fb489db2064920b061cc9615bae672] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=46.1 K 2024-12-09T01:41:08,924 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:08,924 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/B is initiating minor compaction (all files) 2024-12-09T01:41:08,924 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/B in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:08,925 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/61616af18917496b96cf1f059ebdf3a9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/e61ec1ad015e457fa483a20f280db760, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/fb376b7847244b068726dba10f721072] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=36.6 K 2024-12-09T01:41:08,925 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 61616af18917496b96cf1f059ebdf3a9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733708465721 2024-12-09T01:41:08,925 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c7860baf5da452bb1d2293330403d79, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733708465721 2024-12-09T01:41:08,926 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting e61ec1ad015e457fa483a20f280db760, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733708468013 2024-12-09T01:41:08,926 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting a18c98172af7451b826ed629d0356c2e, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733708468013 2024-12-09T01:41:08,926 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting fb376b7847244b068726dba10f721072, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733708468098 2024-12-09T01:41:08,926 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43fb489db2064920b061cc9615bae672, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733708468098 2024-12-09T01:41:08,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:08,929 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T01:41:08,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:08,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:08,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:08,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:08,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:08,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:08,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/47545c7762964bb2953ca20cddab98af is 50, key is test_row_0/A:col10/1733708468791/Put/seqid=0 2024-12-09T01:41:08,958 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#A#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:08,958 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#B#compaction#62 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:08,959 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/866c3da1c3fc434292a7b7621c3c20d5 is 50, key is test_row_0/B:col10/1733708468098/Put/seqid=0 2024-12-09T01:41:08,960 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/3f0c15cb75394ac4a25a4fe5a52e3980 is 50, key is test_row_0/A:col10/1733708468098/Put/seqid=0 2024-12-09T01:41:08,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741897_1073 (size=12301) 2024-12-09T01:41:08,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741898_1074 (size=12983) 2024-12-09T01:41:08,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741899_1075 (size=12983) 2024-12-09T01:41:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T01:41:08,996 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/866c3da1c3fc434292a7b7621c3c20d5 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/866c3da1c3fc434292a7b7621c3c20d5 2024-12-09T01:41:09,000 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/3f0c15cb75394ac4a25a4fe5a52e3980 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/3f0c15cb75394ac4a25a4fe5a52e3980 2024-12-09T01:41:09,010 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/A of 88980a4de80c5830889346818652c3a7 into 3f0c15cb75394ac4a25a4fe5a52e3980(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:09,010 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/B of 88980a4de80c5830889346818652c3a7 into 866c3da1c3fc434292a7b7621c3c20d5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:09,010 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:09,010 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:09,010 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/A, priority=13, startTime=1733708468922; duration=0sec 2024-12-09T01:41:09,010 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/B, priority=13, startTime=1733708468922; duration=0sec 2024-12-09T01:41:09,010 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:09,010 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:A 2024-12-09T01:41:09,010 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:09,011 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:09,011 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:B 2024-12-09T01:41:09,015 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:09,015 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/C is initiating minor compaction (all files) 2024-12-09T01:41:09,015 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/C in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:09,015 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/3b574b2b58e84ae494e04067ece9c1f1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/2165e34413384c51a4d3eb5b3d9cf370, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/c4da6552ec6d45979faceedb34848577] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=36.6 K 2024-12-09T01:41:09,015 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b574b2b58e84ae494e04067ece9c1f1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1733708465721 2024-12-09T01:41:09,016 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2165e34413384c51a4d3eb5b3d9cf370, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1733708468013 2024-12-09T01:41:09,017 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4da6552ec6d45979faceedb34848577, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733708468098 2024-12-09T01:41:09,032 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#C#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:09,033 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/540b5ec51f604e87baefca90fe327236 is 50, key is test_row_0/C:col10/1733708468098/Put/seqid=0 2024-12-09T01:41:09,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-09T01:41:09,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:09,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:09,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:09,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:09,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:09,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:09,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741900_1076 (size=12983) 2024-12-09T01:41:09,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:09,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708529052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:09,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:09,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:09,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:09,065 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/540b5ec51f604e87baefca90fe327236 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/540b5ec51f604e87baefca90fe327236 2024-12-09T01:41:09,074 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/C of 88980a4de80c5830889346818652c3a7 into 540b5ec51f604e87baefca90fe327236(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:09,074 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:09,074 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/C, priority=13, startTime=1733708468922; duration=0sec 2024-12-09T01:41:09,074 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:09,074 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:C 2024-12-09T01:41:09,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708529114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:09,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:09,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:09,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:09,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:09,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708529123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:09,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:09,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:09,128 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:09,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708529153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:09,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:09,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708529153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:09,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:09,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:09,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:09,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:09,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708529163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:09,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:09,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:09,166 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:09,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-09T01:41:09,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:09,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:09,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:09,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:09,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:09,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T01:41:09,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-09T01:41:09,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:09,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:09,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:09,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:09,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:09,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/47545c7762964bb2953ca20cddab98af 2024-12-09T01:41:09,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708529373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:09,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:09,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/a8f97eed45cd40c594669a110e55c305 is 50, key is test_row_0/B:col10/1733708468791/Put/seqid=0 2024-12-09T01:41:09,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741901_1077 (size=12301) 2024-12-09T01:41:09,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:09,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708529433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:09,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:09,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:09,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:09,436 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708529433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:09,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:09,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:09,437 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:09,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-09T01:41:09,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:09,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:09,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:09,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:09,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T01:41:09,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-09T01:41:09,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:09,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:09,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:09,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:09,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:09,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708529683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:09,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:09,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:09,689 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:09,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/a8f97eed45cd40c594669a110e55c305 2024-12-09T01:41:09,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/1e406b66a3da40c4860d7db43c8090f2 is 50, key is test_row_0/C:col10/1733708468791/Put/seqid=0 2024-12-09T01:41:09,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-09T01:41:09,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:09,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:09,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:09,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:09,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:09,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:09,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741902_1078 (size=12301) 2024-12-09T01:41:09,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/1e406b66a3da40c4860d7db43c8090f2 2024-12-09T01:41:09,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/47545c7762964bb2953ca20cddab98af as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/47545c7762964bb2953ca20cddab98af 2024-12-09T01:41:09,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/47545c7762964bb2953ca20cddab98af, entries=150, sequenceid=342, filesize=12.0 K 2024-12-09T01:41:09,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/a8f97eed45cd40c594669a110e55c305 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/a8f97eed45cd40c594669a110e55c305 2024-12-09T01:41:09,837 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/a8f97eed45cd40c594669a110e55c305, entries=150, sequenceid=342, filesize=12.0 K 2024-12-09T01:41:09,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/1e406b66a3da40c4860d7db43c8090f2 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/1e406b66a3da40c4860d7db43c8090f2 2024-12-09T01:41:09,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/1e406b66a3da40c4860d7db43c8090f2, entries=150, sequenceid=342, filesize=12.0 K 2024-12-09T01:41:09,846 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 88980a4de80c5830889346818652c3a7 in 917ms, sequenceid=342, compaction requested=false 2024-12-09T01:41:09,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:09,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:09,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T01:41:09,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:09,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:09,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:09,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:09,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:09,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:09,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/787853de78974f4b83e42e192fa0e01b is 50, key is test_row_0/A:col10/1733708469048/Put/seqid=0 2024-12-09T01:41:09,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-09T01:41:09,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:09,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:09,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:09,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:09,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:09,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:09,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741903_1079 (size=14741) 2024-12-09T01:41:09,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/787853de78974f4b83e42e192fa0e01b 2024-12-09T01:41:09,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/f54205f4d2444943abc2370596fd7304 is 50, key is test_row_0/B:col10/1733708469048/Put/seqid=0 2024-12-09T01:41:09,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741904_1080 (size=12301) 2024-12-09T01:41:09,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/f54205f4d2444943abc2370596fd7304 2024-12-09T01:41:10,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:10,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708529993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:10,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:10,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:10,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708529993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:10,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,002 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:10,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/24869ba34af643009caed653b16d6b6e is 50, key is test_row_0/C:col10/1733708469048/Put/seqid=0 2024-12-09T01:41:10,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741905_1081 (size=12301) 2024-12-09T01:41:10,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T01:41:10,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:10,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708530104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:10,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:10,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708530114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:10,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-09T01:41:10,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:10,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,117 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:10,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:10,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:10,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:10,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:10,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:10,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708530174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:10,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:10,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708530174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:10,179 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_2' in 88980a4de80c5830889346818652c3a7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:41:10,179 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_0' in 88980a4de80c5830889346818652c3a7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:41:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:10,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:10,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708530193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:10,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:10,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-09T01:41:10,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:10,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:10,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:10,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:10,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:10,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:10,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:10,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708530313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:10,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:10,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708530324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:10,332 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:10,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/24869ba34af643009caed653b16d6b6e 2024-12-09T01:41:10,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/787853de78974f4b83e42e192fa0e01b as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/787853de78974f4b83e42e192fa0e01b 2024-12-09T01:41:10,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-09T01:41:10,426 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/787853de78974f4b83e42e192fa0e01b, entries=200, sequenceid=364, filesize=14.4 K 2024-12-09T01:41:10,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:10,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:10,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:10,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:10,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:10,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/f54205f4d2444943abc2370596fd7304 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/f54205f4d2444943abc2370596fd7304 2024-12-09T01:41:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:10,439 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/f54205f4d2444943abc2370596fd7304, entries=150, sequenceid=364, filesize=12.0 K 2024-12-09T01:41:10,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/24869ba34af643009caed653b16d6b6e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/24869ba34af643009caed653b16d6b6e 2024-12-09T01:41:10,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/24869ba34af643009caed653b16d6b6e, entries=150, sequenceid=364, filesize=12.0 K 2024-12-09T01:41:10,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 88980a4de80c5830889346818652c3a7 in 501ms, sequenceid=364, compaction requested=true 2024-12-09T01:41:10,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:10,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:41:10,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:10,449 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:10,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:41:10,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:10,449 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:10,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:41:10,449 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:10,451 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:10,451 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:10,451 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/B is initiating minor compaction (all files) 2024-12-09T01:41:10,451 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/A is initiating minor compaction (all files) 2024-12-09T01:41:10,451 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/B in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:10,451 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/A in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:10,451 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/866c3da1c3fc434292a7b7621c3c20d5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/a8f97eed45cd40c594669a110e55c305, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/f54205f4d2444943abc2370596fd7304] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=36.7 K 2024-12-09T01:41:10,451 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/3f0c15cb75394ac4a25a4fe5a52e3980, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/47545c7762964bb2953ca20cddab98af, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/787853de78974f4b83e42e192fa0e01b] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=39.1 K 2024-12-09T01:41:10,451 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 866c3da1c3fc434292a7b7621c3c20d5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733708468098 2024-12-09T01:41:10,452 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f0c15cb75394ac4a25a4fe5a52e3980, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733708468098 2024-12-09T01:41:10,452 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting a8f97eed45cd40c594669a110e55c305, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733708468789 2024-12-09T01:41:10,452 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47545c7762964bb2953ca20cddab98af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733708468789 2024-12-09T01:41:10,452 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting f54205f4d2444943abc2370596fd7304, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733708469039 2024-12-09T01:41:10,453 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 787853de78974f4b83e42e192fa0e01b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733708469039 2024-12-09T01:41:10,477 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#B#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:10,478 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/b75fd758a3ad4ec1bfd0b663249f5aa7 is 50, key is test_row_0/B:col10/1733708469048/Put/seqid=0 2024-12-09T01:41:10,481 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#A#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:10,481 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/8eb859a81cde4e61a78e5d4d96d18200 is 50, key is test_row_0/A:col10/1733708469048/Put/seqid=0 2024-12-09T01:41:10,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741906_1082 (size=13085) 2024-12-09T01:41:10,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741907_1083 (size=13085) 2024-12-09T01:41:10,500 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/b75fd758a3ad4ec1bfd0b663249f5aa7 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/b75fd758a3ad4ec1bfd0b663249f5aa7 2024-12-09T01:41:10,502 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/8eb859a81cde4e61a78e5d4d96d18200 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/8eb859a81cde4e61a78e5d4d96d18200 2024-12-09T01:41:10,512 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/B of 88980a4de80c5830889346818652c3a7 into b75fd758a3ad4ec1bfd0b663249f5aa7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:10,512 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:10,513 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/B, priority=13, startTime=1733708470449; duration=0sec 2024-12-09T01:41:10,513 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:10,513 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:B 2024-12-09T01:41:10,513 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:10,515 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:10,515 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/C is initiating minor compaction (all files) 2024-12-09T01:41:10,515 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/C in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:10,515 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/540b5ec51f604e87baefca90fe327236, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/1e406b66a3da40c4860d7db43c8090f2, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/24869ba34af643009caed653b16d6b6e] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=36.7 K 2024-12-09T01:41:10,516 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 540b5ec51f604e87baefca90fe327236, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733708468098 2024-12-09T01:41:10,516 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e406b66a3da40c4860d7db43c8090f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1733708468789 2024-12-09T01:41:10,517 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 24869ba34af643009caed653b16d6b6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733708469039 2024-12-09T01:41:10,517 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/A of 88980a4de80c5830889346818652c3a7 into 8eb859a81cde4e61a78e5d4d96d18200(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:10,517 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:10,517 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/A, priority=13, startTime=1733708470449; duration=0sec 2024-12-09T01:41:10,517 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:10,517 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:A 2024-12-09T01:41:10,527 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#C#compaction#71 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:10,528 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/1231e3887709433cb1420064b692ab3c is 50, key is test_row_0/C:col10/1733708469048/Put/seqid=0 2024-12-09T01:41:10,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741908_1084 (size=13085) 2024-12-09T01:41:10,564 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/1231e3887709433cb1420064b692ab3c as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/1231e3887709433cb1420064b692ab3c 2024-12-09T01:41:10,577 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/C of 88980a4de80c5830889346818652c3a7 into 1231e3887709433cb1420064b692ab3c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:10,577 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:10,577 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/C, priority=13, startTime=1733708470449; duration=0sec 2024-12-09T01:41:10,578 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:10,578 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:C 2024-12-09T01:41:10,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-12-09T01:41:10,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:10,583 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T01:41:10,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:10,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:10,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:10,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:10,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:10,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:10,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/b902f4d2f75949dcae4a870d0f5aed64 is 50, key is test_row_0/A:col10/1733708469964/Put/seqid=0 2024-12-09T01:41:10,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741909_1085 (size=12301) 2024-12-09T01:41:10,600 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/b902f4d2f75949dcae4a870d0f5aed64 2024-12-09T01:41:10,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/2ec5d3bcdb1742d2990c7d8aa031192b is 50, key is test_row_0/B:col10/1733708469964/Put/seqid=0 2024-12-09T01:41:10,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:10,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:10,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741910_1086 (size=12301) 2024-12-09T01:41:10,634 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/2ec5d3bcdb1742d2990c7d8aa031192b 2024-12-09T01:41:10,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/d2b0656bafee4174afa94ca6d6c85d21 is 50, key is test_row_0/C:col10/1733708469964/Put/seqid=0 2024-12-09T01:41:10,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741911_1087 (size=12301) 2024-12-09T01:41:10,679 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/d2b0656bafee4174afa94ca6d6c85d21 2024-12-09T01:41:10,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/b902f4d2f75949dcae4a870d0f5aed64 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/b902f4d2f75949dcae4a870d0f5aed64 2024-12-09T01:41:10,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:10,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708530691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:10,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:10,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708530694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:10,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:10,701 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/b902f4d2f75949dcae4a870d0f5aed64, entries=150, sequenceid=380, filesize=12.0 K 2024-12-09T01:41:10,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/2ec5d3bcdb1742d2990c7d8aa031192b as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/2ec5d3bcdb1742d2990c7d8aa031192b 2024-12-09T01:41:10,709 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/2ec5d3bcdb1742d2990c7d8aa031192b, entries=150, sequenceid=380, filesize=12.0 K 2024-12-09T01:41:10,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/d2b0656bafee4174afa94ca6d6c85d21 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/d2b0656bafee4174afa94ca6d6c85d21 2024-12-09T01:41:10,716 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/d2b0656bafee4174afa94ca6d6c85d21, entries=150, sequenceid=380, filesize=12.0 K 2024-12-09T01:41:10,717 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 88980a4de80c5830889346818652c3a7 in 134ms, sequenceid=380, compaction requested=false 2024-12-09T01:41:10,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:10,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:10,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-09T01:41:10,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-09T01:41:10,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-09T01:41:10,723 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8400 sec 2024-12-09T01:41:10,725 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.8500 sec 2024-12-09T01:41:10,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:10,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T01:41:10,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:10,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:10,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:10,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:10,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:10,807 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:10,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/825e790e83474419bf191e90a3ee9e47 is 50, key is test_row_0/A:col10/1733708470692/Put/seqid=0 2024-12-09T01:41:10,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741912_1088 (size=14741) 2024-12-09T01:41:10,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:10,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708530858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:10,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,863 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:10,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708530860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:10,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:10,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:10,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708530975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:10,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:10,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:10,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708530984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:10,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:11,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-09T01:41:11,044 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:41:11,044 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:41:11,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-12-09T01:41:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T01:41:11,047 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:41:11,049 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:41:11,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:41:11,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T01:41:11,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:11,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708531184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:11,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:11,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:11,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:11,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:11,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708531194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:11,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:11,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:11,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:11,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T01:41:11,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:11,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:11,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:11,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:11,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:11,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:11,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:11,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708531213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:11,216 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_0' in 88980a4de80c5830889346818652c3a7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:41:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:11,217 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:11,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/825e790e83474419bf191e90a3ee9e47 2024-12-09T01:41:11,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/a3b531ac61424b32be1f9c73566719f5 is 50, key is test_row_0/B:col10/1733708470692/Put/seqid=0 2024-12-09T01:41:11,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741913_1089 (size=12301) 2024-12-09T01:41:11,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T01:41:11,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:11,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:11,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:11,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:11,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:11,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T01:41:11,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:11,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708531494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:11,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:11,495 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:11,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:11,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708531503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:11,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:11,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:11,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:11,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T01:41:11,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:11,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:11,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:11,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:11,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:11,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:11,643 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/a3b531ac61424b32be1f9c73566719f5 2024-12-09T01:41:11,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/2c78734d4116417fb42c53ba6482ee1a is 50, key is test_row_0/C:col10/1733708470692/Put/seqid=0 2024-12-09T01:41:11,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741914_1090 (size=12301) 2024-12-09T01:41:11,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/2c78734d4116417fb42c53ba6482ee1a 2024-12-09T01:41:11,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/825e790e83474419bf191e90a3ee9e47 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/825e790e83474419bf191e90a3ee9e47 2024-12-09T01:41:11,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T01:41:11,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:11,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:11,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:11,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:11,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:11,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:11,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/825e790e83474419bf191e90a3ee9e47, entries=200, sequenceid=401, filesize=14.4 K 2024-12-09T01:41:11,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/a3b531ac61424b32be1f9c73566719f5 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/a3b531ac61424b32be1f9c73566719f5 2024-12-09T01:41:11,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T01:41:11,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/a3b531ac61424b32be1f9c73566719f5, entries=150, sequenceid=401, filesize=12.0 K 2024-12-09T01:41:11,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/2c78734d4116417fb42c53ba6482ee1a as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/2c78734d4116417fb42c53ba6482ee1a 2024-12-09T01:41:11,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/2c78734d4116417fb42c53ba6482ee1a, entries=150, sequenceid=401, filesize=12.0 K 2024-12-09T01:41:11,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 88980a4de80c5830889346818652c3a7 in 885ms, sequenceid=401, compaction requested=true 2024-12-09T01:41:11,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:11,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:41:11,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:11,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:41:11,691 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:11,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:11,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:41:11,691 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T01:41:11,691 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:11,693 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:11,693 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/B is initiating minor compaction (all files) 2024-12-09T01:41:11,693 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:11,693 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/B in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:11,693 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/A is initiating minor compaction (all files) 2024-12-09T01:41:11,693 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/b75fd758a3ad4ec1bfd0b663249f5aa7, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/2ec5d3bcdb1742d2990c7d8aa031192b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/a3b531ac61424b32be1f9c73566719f5] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=36.8 K 2024-12-09T01:41:11,693 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/A in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:11,693 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/8eb859a81cde4e61a78e5d4d96d18200, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/b902f4d2f75949dcae4a870d0f5aed64, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/825e790e83474419bf191e90a3ee9e47] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=39.2 K 2024-12-09T01:41:11,694 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting b75fd758a3ad4ec1bfd0b663249f5aa7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733708469039 2024-12-09T01:41:11,694 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8eb859a81cde4e61a78e5d4d96d18200, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733708469039 2024-12-09T01:41:11,694 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ec5d3bcdb1742d2990c7d8aa031192b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733708469964 2024-12-09T01:41:11,695 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting a3b531ac61424b32be1f9c73566719f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733708470685 2024-12-09T01:41:11,695 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting b902f4d2f75949dcae4a870d0f5aed64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733708469964 2024-12-09T01:41:11,697 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 825e790e83474419bf191e90a3ee9e47, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733708470685 2024-12-09T01:41:11,712 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#B#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:11,713 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#A#compaction#78 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:11,713 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/27444b86b0bc4c85bbc6b4b63ff154f4 is 50, key is test_row_0/B:col10/1733708470692/Put/seqid=0 2024-12-09T01:41:11,714 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/05074c791b0e476987d44bfde5793996 is 50, key is test_row_0/A:col10/1733708470692/Put/seqid=0 2024-12-09T01:41:11,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741915_1091 (size=13187) 2024-12-09T01:41:11,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741916_1092 (size=13187) 2024-12-09T01:41:11,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-12-09T01:41:11,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:11,823 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T01:41:11,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:11,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:11,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:11,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:11,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:11,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:11,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/4942f7209839421885f2e1c59a15fffc is 50, key is test_row_0/A:col10/1733708470830/Put/seqid=0 2024-12-09T01:41:11,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741917_1093 (size=12301) 2024-12-09T01:41:12,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:12,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:12,104 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708472037, max timestamp=1733708472100Num uniques 859; 2024-12-09T01:41:12,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:41:12,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708472037, max timestamp=1733708472100Num uniques 859; 2024-12-09T01:41:12,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708472037, max timestamp=1733708472100Num uniques 859; 2024-12-09T01:41:12,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:41:12,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:41:12,159 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/27444b86b0bc4c85bbc6b4b63ff154f4 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/27444b86b0bc4c85bbc6b4b63ff154f4 2024-12-09T01:41:12,160 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/05074c791b0e476987d44bfde5793996 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/05074c791b0e476987d44bfde5793996 2024-12-09T01:41:12,171 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/A of 88980a4de80c5830889346818652c3a7 into 05074c791b0e476987d44bfde5793996(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:12,171 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:12,172 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/A, priority=13, startTime=1733708471691; duration=0sec 2024-12-09T01:41:12,172 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:12,172 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:A 2024-12-09T01:41:12,172 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:12,176 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:12,176 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/C is initiating minor compaction (all files) 2024-12-09T01:41:12,176 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/C in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:12,176 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/1231e3887709433cb1420064b692ab3c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/d2b0656bafee4174afa94ca6d6c85d21, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/2c78734d4116417fb42c53ba6482ee1a] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=36.8 K 2024-12-09T01:41:12,177 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/B of 88980a4de80c5830889346818652c3a7 into 27444b86b0bc4c85bbc6b4b63ff154f4(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:12,177 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:12,177 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/B, priority=13, startTime=1733708471691; duration=0sec 2024-12-09T01:41:12,178 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:12,178 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:B 2024-12-09T01:41:12,178 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1231e3887709433cb1420064b692ab3c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733708469039 2024-12-09T01:41:12,179 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2b0656bafee4174afa94ca6d6c85d21, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1733708469964 2024-12-09T01:41:12,180 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c78734d4116417fb42c53ba6482ee1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733708470685 2024-12-09T01:41:12,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T01:41:12,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708532182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:12,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:12,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708532190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:12,200 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#C#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:12,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:12,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708532194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:12,202 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/643e5d5c827a49968e85bf81190993ac is 50, key is test_row_0/C:col10/1733708470692/Put/seqid=0 2024-12-09T01:41:12,202 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_2' in 88980a4de80c5830889346818652c3a7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:41:12,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:12,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:12,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:12,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:12,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708532205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:12,209 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_0' in 88980a4de80c5830889346818652c3a7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:41:12,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:12,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:12,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:12,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741918_1094 (size=13187) 2024-12-09T01:41:12,242 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/4942f7209839421885f2e1c59a15fffc 2024-12-09T01:41:12,260 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/643e5d5c827a49968e85bf81190993ac as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/643e5d5c827a49968e85bf81190993ac 2024-12-09T01:41:12,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/51bbccb1cf1448d794796da6359b6216 is 50, key is test_row_0/B:col10/1733708470830/Put/seqid=0 2024-12-09T01:41:12,282 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/C of 88980a4de80c5830889346818652c3a7 into 643e5d5c827a49968e85bf81190993ac(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:12,282 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:12,282 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/C, priority=13, startTime=1733708471691; duration=0sec 2024-12-09T01:41:12,282 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:12,282 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:C 2024-12-09T01:41:12,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:12,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708532293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:12,297 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:12,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:12,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708532304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:12,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:12,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:12,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:12,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741919_1095 (size=12301) 2024-12-09T01:41:12,319 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/51bbccb1cf1448d794796da6359b6216 2024-12-09T01:41:12,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/3c60ff22a79b4ea6b7312bc93efbcda6 is 50, key is test_row_0/C:col10/1733708470830/Put/seqid=0 2024-12-09T01:41:12,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741920_1096 (size=12301) 2024-12-09T01:41:12,387 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=412 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/3c60ff22a79b4ea6b7312bc93efbcda6 2024-12-09T01:41:12,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/4942f7209839421885f2e1c59a15fffc as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/4942f7209839421885f2e1c59a15fffc 2024-12-09T01:41:12,404 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/4942f7209839421885f2e1c59a15fffc, entries=150, sequenceid=412, filesize=12.0 K 2024-12-09T01:41:12,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/51bbccb1cf1448d794796da6359b6216 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/51bbccb1cf1448d794796da6359b6216 2024-12-09T01:41:12,419 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/51bbccb1cf1448d794796da6359b6216, entries=150, sequenceid=412, filesize=12.0 K 2024-12-09T01:41:12,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/3c60ff22a79b4ea6b7312bc93efbcda6 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/3c60ff22a79b4ea6b7312bc93efbcda6 2024-12-09T01:41:12,431 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/3c60ff22a79b4ea6b7312bc93efbcda6, entries=150, sequenceid=412, filesize=12.0 K 2024-12-09T01:41:12,436 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 88980a4de80c5830889346818652c3a7 in 613ms, sequenceid=412, compaction requested=false 2024-12-09T01:41:12,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:12,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:12,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-09T01:41:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-09T01:41:12,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-09T01:41:12,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3890 sec 2024-12-09T01:41:12,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.4000 sec 2024-12-09T01:41:12,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T01:41:12,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:12,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:12,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:12,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:12,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:12,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:12,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/3a9441b5dda54cc1bb8c5a0b176ea3c9 is 50, key is test_row_0/A:col10/1733708472185/Put/seqid=0 2024-12-09T01:41:12,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741921_1097 (size=14741) 2024-12-09T01:41:12,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/3a9441b5dda54cc1bb8c5a0b176ea3c9 2024-12-09T01:41:12,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708532551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:12,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:12,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:12,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:12,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708532556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:12,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:12,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:12,559 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:12,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/18513099bd58488a9bcae93cc3199ee3 is 50, key is test_row_0/B:col10/1733708472185/Put/seqid=0 2024-12-09T01:41:12,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741922_1098 (size=12301) 2024-12-09T01:41:12,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/18513099bd58488a9bcae93cc3199ee3 2024-12-09T01:41:12,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/87cee428298449e5b21a0c388236256d is 50, key is test_row_0/C:col10/1733708472185/Put/seqid=0 2024-12-09T01:41:12,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741923_1099 (size=12301) 2024-12-09T01:41:12,660 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/87cee428298449e5b21a0c388236256d 2024-12-09T01:41:12,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708532664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:12,670 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708532664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:12,670 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:12,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/3a9441b5dda54cc1bb8c5a0b176ea3c9 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/3a9441b5dda54cc1bb8c5a0b176ea3c9 2024-12-09T01:41:12,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/3a9441b5dda54cc1bb8c5a0b176ea3c9, entries=200, sequenceid=451, filesize=14.4 K 2024-12-09T01:41:12,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/18513099bd58488a9bcae93cc3199ee3 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/18513099bd58488a9bcae93cc3199ee3 2024-12-09T01:41:12,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/18513099bd58488a9bcae93cc3199ee3, entries=150, sequenceid=451, filesize=12.0 K 2024-12-09T01:41:12,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/87cee428298449e5b21a0c388236256d as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/87cee428298449e5b21a0c388236256d 2024-12-09T01:41:12,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/87cee428298449e5b21a0c388236256d, entries=150, sequenceid=451, filesize=12.0 K 2024-12-09T01:41:12,714 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 88980a4de80c5830889346818652c3a7 in 201ms, sequenceid=451, compaction requested=true 2024-12-09T01:41:12,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:12,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:41:12,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:12,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:41:12,714 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:12,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:12,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:41:12,714 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T01:41:12,714 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:12,716 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:12,716 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/A is initiating minor compaction (all files) 2024-12-09T01:41:12,716 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/A in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:12,716 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/05074c791b0e476987d44bfde5793996, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/4942f7209839421885f2e1c59a15fffc, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/3a9441b5dda54cc1bb8c5a0b176ea3c9] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=39.3 K 2024-12-09T01:41:12,716 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:12,716 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05074c791b0e476987d44bfde5793996, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733708470685 2024-12-09T01:41:12,717 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/B is initiating minor compaction (all files) 2024-12-09T01:41:12,717 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/B in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:12,717 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/27444b86b0bc4c85bbc6b4b63ff154f4, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/51bbccb1cf1448d794796da6359b6216, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/18513099bd58488a9bcae93cc3199ee3] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=36.9 K 2024-12-09T01:41:12,718 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 27444b86b0bc4c85bbc6b4b63ff154f4, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733708470685 2024-12-09T01:41:12,718 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 51bbccb1cf1448d794796da6359b6216, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733708470816 2024-12-09T01:41:12,719 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 18513099bd58488a9bcae93cc3199ee3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733708472177 2024-12-09T01:41:12,719 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4942f7209839421885f2e1c59a15fffc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733708470816 2024-12-09T01:41:12,721 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a9441b5dda54cc1bb8c5a0b176ea3c9, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733708472177 2024-12-09T01:41:12,734 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#A#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:12,735 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/796865ff55654e3a812d67bf6708d119 is 50, key is test_row_0/A:col10/1733708472185/Put/seqid=0 2024-12-09T01:41:12,741 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#B#compaction#88 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:12,741 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/2bf6c65c227643cab788324f3144cff6 is 50, key is test_row_0/B:col10/1733708472185/Put/seqid=0 2024-12-09T01:41:12,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741924_1100 (size=13289) 2024-12-09T01:41:12,771 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/796865ff55654e3a812d67bf6708d119 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/796865ff55654e3a812d67bf6708d119 2024-12-09T01:41:12,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741925_1101 (size=13289) 2024-12-09T01:41:12,783 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/A of 88980a4de80c5830889346818652c3a7 into 796865ff55654e3a812d67bf6708d119(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:12,783 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:12,784 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/A, priority=13, startTime=1733708472714; duration=0sec 2024-12-09T01:41:12,784 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:12,784 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:A 2024-12-09T01:41:12,784 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:12,786 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:12,786 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/C is initiating minor compaction (all files) 2024-12-09T01:41:12,786 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/C in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:12,786 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/643e5d5c827a49968e85bf81190993ac, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/3c60ff22a79b4ea6b7312bc93efbcda6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/87cee428298449e5b21a0c388236256d] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=36.9 K 2024-12-09T01:41:12,787 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 643e5d5c827a49968e85bf81190993ac, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733708470685 2024-12-09T01:41:12,788 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c60ff22a79b4ea6b7312bc93efbcda6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=412, earliestPutTs=1733708470816 2024-12-09T01:41:12,788 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87cee428298449e5b21a0c388236256d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733708472177 2024-12-09T01:41:12,803 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#C#compaction#89 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:12,803 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/8567cec77b0f4a11a315f2709431865f is 50, key is test_row_0/C:col10/1733708472185/Put/seqid=0 2024-12-09T01:41:12,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741926_1102 (size=13289) 2024-12-09T01:41:12,866 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/8567cec77b0f4a11a315f2709431865f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/8567cec77b0f4a11a315f2709431865f 2024-12-09T01:41:12,878 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/C of 88980a4de80c5830889346818652c3a7 into 8567cec77b0f4a11a315f2709431865f(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:12,878 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:12,878 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/C, priority=13, startTime=1733708472714; duration=0sec 2024-12-09T01:41:12,879 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:12,879 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:C 2024-12-09T01:41:12,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T01:41:12,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:12,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:12,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:12,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:12,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:12,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:12,882 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:12,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/1950b0682a0949d3a111779b948657d4 is 50, key is test_row_0/A:col10/1733708472551/Put/seqid=0 2024-12-09T01:41:12,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741927_1103 (size=14741) 2024-12-09T01:41:12,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708472938, max timestamp=1733708472964Num uniques 859; 2024-12-09T01:41:12,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:41:12,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708472938, max timestamp=1733708472964Num uniques 859; 2024-12-09T01:41:12,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708472938, max timestamp=1733708472964Num uniques 859; 2024-12-09T01:41:12,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:41:12,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:41:13,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:13,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708533033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:13,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:13,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708533038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:13,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:13,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:13,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:13,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:13,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708533154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:13,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:13,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:13,156 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:13,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:13,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708533154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:13,157 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:13,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:13,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:13,192 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/2bf6c65c227643cab788324f3144cff6 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/2bf6c65c227643cab788324f3144cff6 2024-12-09T01:41:13,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-09T01:41:13,194 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:41:13,194 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:41:13,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-12-09T01:41:13,197 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:41:13,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T01:41:13,198 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:41:13,199 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:41:13,206 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/B of 88980a4de80c5830889346818652c3a7 into 2bf6c65c227643cab788324f3144cff6(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:13,206 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:13,206 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/B, priority=13, startTime=1733708472714; duration=0sec 2024-12-09T01:41:13,206 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:13,206 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:B 2024-12-09T01:41:13,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:13,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708533245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:13,250 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_0' in 88980a4de80c5830889346818652c3a7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:41:13,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:13,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:13,251 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:13,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T01:41:13,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T01:41:13,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:13,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:13,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:13,352 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:13,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:13,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:13,360 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/1950b0682a0949d3a111779b948657d4 2024-12-09T01:41:13,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:13,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708533363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:13,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:13,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:13,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:13,366 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:13,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708533364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:13,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:13,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:13,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:13,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/e546655b162f477f996a1c45b30ed473 is 50, key is test_row_0/B:col10/1733708472551/Put/seqid=0 2024-12-09T01:41:13,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741928_1104 (size=12301) 2024-12-09T01:41:13,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/e546655b162f477f996a1c45b30ed473 2024-12-09T01:41:13,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/77f235daa36645c692eede49532f5364 is 50, key is test_row_0/C:col10/1733708472551/Put/seqid=0 2024-12-09T01:41:13,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741929_1105 (size=12301) 2024-12-09T01:41:13,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T01:41:13,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:13,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:13,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:13,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:13,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:13,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:13,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T01:41:13,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T01:41:13,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:13,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:13,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:13,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:13,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:13,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:13,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:13,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708533673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:13,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:13,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708533674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:13,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:13,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/77f235daa36645c692eede49532f5364 2024-12-09T01:41:13,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/1950b0682a0949d3a111779b948657d4 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/1950b0682a0949d3a111779b948657d4 2024-12-09T01:41:13,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/1950b0682a0949d3a111779b948657d4, entries=200, sequenceid=464, filesize=14.4 K 2024-12-09T01:41:13,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/e546655b162f477f996a1c45b30ed473 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/e546655b162f477f996a1c45b30ed473 2024-12-09T01:41:13,822 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/e546655b162f477f996a1c45b30ed473, entries=150, sequenceid=464, filesize=12.0 K 2024-12-09T01:41:13,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T01:41:13,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:13,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:13,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:13,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:13,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/77f235daa36645c692eede49532f5364 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/77f235daa36645c692eede49532f5364 2024-12-09T01:41:13,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:13,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T01:41:13,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:13,829 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/77f235daa36645c692eede49532f5364, entries=150, sequenceid=464, filesize=12.0 K 2024-12-09T01:41:13,830 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 88980a4de80c5830889346818652c3a7 in 948ms, sequenceid=464, compaction requested=false 2024-12-09T01:41:13,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:13,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-12-09T01:41:13,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:13,980 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-12-09T01:41:13,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:13,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:13,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:13,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:13,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:13,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:13,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/b1f2ad8a58da4d248b8bed7d8dd0767c is 50, key is test_row_0/A:col10/1733708473035/Put/seqid=0 2024-12-09T01:41:14,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741930_1106 (size=12301) 2024-12-09T01:41:14,018 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/b1f2ad8a58da4d248b8bed7d8dd0767c 2024-12-09T01:41:14,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/361feef197c44827b585a2c6b6a70827 is 50, key is test_row_0/B:col10/1733708473035/Put/seqid=0 2024-12-09T01:41:14,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741931_1107 (size=12301) 2024-12-09T01:41:14,055 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/361feef197c44827b585a2c6b6a70827 2024-12-09T01:41:14,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/1090b4a26c2447479a42c4c122e03470 is 50, key is test_row_0/C:col10/1733708473035/Put/seqid=0 2024-12-09T01:41:14,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741932_1108 (size=12301) 2024-12-09T01:41:14,077 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=499 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/1090b4a26c2447479a42c4c122e03470 2024-12-09T01:41:14,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/b1f2ad8a58da4d248b8bed7d8dd0767c as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/b1f2ad8a58da4d248b8bed7d8dd0767c 2024-12-09T01:41:14,119 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/b1f2ad8a58da4d248b8bed7d8dd0767c, entries=150, sequenceid=499, filesize=12.0 K 2024-12-09T01:41:14,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/361feef197c44827b585a2c6b6a70827 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/361feef197c44827b585a2c6b6a70827 2024-12-09T01:41:14,126 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/361feef197c44827b585a2c6b6a70827, entries=150, sequenceid=499, filesize=12.0 K 2024-12-09T01:41:14,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/1090b4a26c2447479a42c4c122e03470 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/1090b4a26c2447479a42c4c122e03470 2024-12-09T01:41:14,144 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/1090b4a26c2447479a42c4c122e03470, entries=150, sequenceid=499, filesize=12.0 K 2024-12-09T01:41:14,148 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=0 B/0 for 88980a4de80c5830889346818652c3a7 in 168ms, sequenceid=499, compaction requested=true 2024-12-09T01:41:14,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:14,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:14,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-09T01:41:14,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-12-09T01:41:14,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-12-09T01:41:14,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 951 msec 2024-12-09T01:41:14,155 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 959 msec 2024-12-09T01:41:14,201 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:41:14,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:14,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:14,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:14,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:14,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:14,202 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:14,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:14,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/429ebb2636a048c689c27cc03b9463de is 50, key is test_row_0/A:col10/1733708474193/Put/seqid=0 2024-12-09T01:41:14,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741933_1109 (size=17181) 2024-12-09T01:41:14,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/429ebb2636a048c689c27cc03b9463de 2024-12-09T01:41:14,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/103cb277da454f49b6b951eaa1493875 is 50, key is test_row_0/B:col10/1733708474193/Put/seqid=0 2024-12-09T01:41:14,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741934_1110 (size=12301) 2024-12-09T01:41:14,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-09T01:41:14,334 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:41:14,335 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:41:14,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-12-09T01:41:14,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T01:41:14,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708474319, max timestamp=1733708474340Num uniques 859; 2024-12-09T01:41:14,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708474319, max timestamp=1733708474340Num uniques 859; 2024-12-09T01:41:14,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:41:14,346 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:41:14,346 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:41:14,347 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:41:14,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:41:14,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708474319, max timestamp=1733708474340Num uniques 859; 2024-12-09T01:41:14,349 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:41:14,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:14,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708534421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:14,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:14,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708534423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:14,430 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:14,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T01:41:14,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T01:41:14,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:14,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:14,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:14,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:14,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:14,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:14,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708534534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:14,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:14,540 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:14,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708534534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:14,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:14,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:14,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:14,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T01:41:14,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T01:41:14,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:14,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:14,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:14,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:14,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:14,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:14,699 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/103cb277da454f49b6b951eaa1493875 2024-12-09T01:41:14,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/c5c172e5984f46dfa8b070b2487f93a2 is 50, key is test_row_0/C:col10/1733708474193/Put/seqid=0 2024-12-09T01:41:14,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:14,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708534743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:14,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:14,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708534744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:14,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:14,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741935_1111 (size=12301) 2024-12-09T01:41:14,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=509 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/c5c172e5984f46dfa8b070b2487f93a2 2024-12-09T01:41:14,803 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/429ebb2636a048c689c27cc03b9463de as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/429ebb2636a048c689c27cc03b9463de 2024-12-09T01:41:14,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T01:41:14,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/429ebb2636a048c689c27cc03b9463de, entries=250, sequenceid=509, filesize=16.8 K 2024-12-09T01:41:14,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:14,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:14,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:14,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:14,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:14,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/103cb277da454f49b6b951eaa1493875 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/103cb277da454f49b6b951eaa1493875 2024-12-09T01:41:14,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:14,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/103cb277da454f49b6b951eaa1493875, entries=150, sequenceid=509, filesize=12.0 K 2024-12-09T01:41:14,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/c5c172e5984f46dfa8b070b2487f93a2 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/c5c172e5984f46dfa8b070b2487f93a2 2024-12-09T01:41:14,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/c5c172e5984f46dfa8b070b2487f93a2, entries=150, sequenceid=509, filesize=12.0 K 2024-12-09T01:41:14,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 88980a4de80c5830889346818652c3a7 in 626ms, sequenceid=509, compaction requested=true 2024-12-09T01:41:14,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:14,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:41:14,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:14,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:41:14,827 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:41:14,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:14,827 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:41:14,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:41:14,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:14,831 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:41:14,831 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/B is initiating minor compaction (all files) 2024-12-09T01:41:14,831 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/B in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:14,831 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/2bf6c65c227643cab788324f3144cff6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/e546655b162f477f996a1c45b30ed473, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/361feef197c44827b585a2c6b6a70827, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/103cb277da454f49b6b951eaa1493875] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=49.0 K 2024-12-09T01:41:14,832 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 2bf6c65c227643cab788324f3144cff6, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733708472177 2024-12-09T01:41:14,838 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57512 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:41:14,838 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/A is initiating minor compaction (all files) 2024-12-09T01:41:14,838 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/A in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:14,839 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/796865ff55654e3a812d67bf6708d119, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/1950b0682a0949d3a111779b948657d4, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/b1f2ad8a58da4d248b8bed7d8dd0767c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/429ebb2636a048c689c27cc03b9463de] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=56.2 K 2024-12-09T01:41:14,839 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting e546655b162f477f996a1c45b30ed473, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1733708472548 2024-12-09T01:41:14,839 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 796865ff55654e3a812d67bf6708d119, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733708472177 2024-12-09T01:41:14,840 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 361feef197c44827b585a2c6b6a70827, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1733708473019 2024-12-09T01:41:14,840 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1950b0682a0949d3a111779b948657d4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1733708472536 2024-12-09T01:41:14,840 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 103cb277da454f49b6b951eaa1493875, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1733708474193 2024-12-09T01:41:14,841 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1f2ad8a58da4d248b8bed7d8dd0767c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1733708473019 2024-12-09T01:41:14,847 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 429ebb2636a048c689c27cc03b9463de, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1733708474190 2024-12-09T01:41:14,872 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#B#compaction#99 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:14,873 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/c512c634069e464a999a22544dd59da6 is 50, key is test_row_0/B:col10/1733708474193/Put/seqid=0 2024-12-09T01:41:14,881 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#A#compaction#100 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:14,882 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/4059d70cab9247e791654c309b73d797 is 50, key is test_row_0/A:col10/1733708474193/Put/seqid=0 2024-12-09T01:41:14,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741936_1112 (size=13425) 2024-12-09T01:41:14,926 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/c512c634069e464a999a22544dd59da6 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/c512c634069e464a999a22544dd59da6 2024-12-09T01:41:14,936 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88980a4de80c5830889346818652c3a7/B of 88980a4de80c5830889346818652c3a7 into c512c634069e464a999a22544dd59da6(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:14,936 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:14,936 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/B, priority=12, startTime=1733708474827; duration=0sec 2024-12-09T01:41:14,937 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:14,937 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:B 2024-12-09T01:41:14,938 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:41:14,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741937_1113 (size=13425) 2024-12-09T01:41:14,942 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:41:14,943 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/C is initiating minor compaction (all files) 2024-12-09T01:41:14,944 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/C in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:14,952 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/4059d70cab9247e791654c309b73d797 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/4059d70cab9247e791654c309b73d797 2024-12-09T01:41:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T01:41:14,964 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/8567cec77b0f4a11a315f2709431865f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/77f235daa36645c692eede49532f5364, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/1090b4a26c2447479a42c4c122e03470, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/c5c172e5984f46dfa8b070b2487f93a2] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=49.0 K 2024-12-09T01:41:14,965 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88980a4de80c5830889346818652c3a7/A of 88980a4de80c5830889346818652c3a7 into 4059d70cab9247e791654c309b73d797(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:14,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-12-09T01:41:14,965 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:14,965 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/A, priority=12, startTime=1733708474827; duration=0sec 2024-12-09T01:41:14,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:14,965 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:14,965 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:A 2024-12-09T01:41:14,965 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-09T01:41:14,965 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 8567cec77b0f4a11a315f2709431865f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1733708472177 2024-12-09T01:41:14,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:14,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:14,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:14,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:14,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:14,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:14,966 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 77f235daa36645c692eede49532f5364, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1733708472548 2024-12-09T01:41:14,967 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 1090b4a26c2447479a42c4c122e03470, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=499, earliestPutTs=1733708473019 2024-12-09T01:41:14,969 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting c5c172e5984f46dfa8b070b2487f93a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1733708474193 2024-12-09T01:41:14,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/ceb54252e8e149dd88b5bb9fb72c7e9c is 50, key is test_row_0/A:col10/1733708474406/Put/seqid=0 2024-12-09T01:41:15,020 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#C#compaction#102 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:15,021 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/bc5f53f6935a4f918f7c05b8ee7d0e78 is 50, key is test_row_0/C:col10/1733708474193/Put/seqid=0 2024-12-09T01:41:15,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741938_1114 (size=12301) 2024-12-09T01:41:15,044 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/ceb54252e8e149dd88b5bb9fb72c7e9c 2024-12-09T01:41:15,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741939_1115 (size=13425) 2024-12-09T01:41:15,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:15,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:15,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/3acf36f60a034bc2909a6368567b2fd6 is 50, key is test_row_0/B:col10/1733708474406/Put/seqid=0 2024-12-09T01:41:15,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708535095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:15,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:15,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708535107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:15,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:15,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741940_1116 (size=12301) 2024-12-09T01:41:15,133 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/3acf36f60a034bc2909a6368567b2fd6 2024-12-09T01:41:15,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/23ee3674e0194a138911397a9ec6f257 is 50, key is test_row_0/C:col10/1733708474406/Put/seqid=0 2024-12-09T01:41:15,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741941_1117 (size=12301) 2024-12-09T01:41:15,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708535214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:15,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:15,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708535223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:15,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:15,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:15,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:15,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708535423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:15,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:15,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:15,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:15,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708535434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:15,450 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T01:41:15,478 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/bc5f53f6935a4f918f7c05b8ee7d0e78 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/bc5f53f6935a4f918f7c05b8ee7d0e78 2024-12-09T01:41:15,488 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88980a4de80c5830889346818652c3a7/C of 88980a4de80c5830889346818652c3a7 into bc5f53f6935a4f918f7c05b8ee7d0e78(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:15,488 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:15,488 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/C, priority=12, startTime=1733708474827; duration=0sec 2024-12-09T01:41:15,488 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:15,488 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:C 2024-12-09T01:41:15,600 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=548 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/23ee3674e0194a138911397a9ec6f257 2024-12-09T01:41:15,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/ceb54252e8e149dd88b5bb9fb72c7e9c as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ceb54252e8e149dd88b5bb9fb72c7e9c 2024-12-09T01:41:15,613 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ceb54252e8e149dd88b5bb9fb72c7e9c, entries=150, sequenceid=548, filesize=12.0 K 2024-12-09T01:41:15,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/3acf36f60a034bc2909a6368567b2fd6 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/3acf36f60a034bc2909a6368567b2fd6 2024-12-09T01:41:15,634 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/3acf36f60a034bc2909a6368567b2fd6, entries=150, sequenceid=548, filesize=12.0 K 2024-12-09T01:41:15,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/23ee3674e0194a138911397a9ec6f257 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/23ee3674e0194a138911397a9ec6f257 2024-12-09T01:41:15,642 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/23ee3674e0194a138911397a9ec6f257, entries=150, sequenceid=548, filesize=12.0 K 2024-12-09T01:41:15,647 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 88980a4de80c5830889346818652c3a7 in 682ms, sequenceid=548, compaction requested=false 2024-12-09T01:41:15,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:15,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:15,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-12-09T01:41:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-12-09T01:41:15,653 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-12-09T01:41:15,653 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3030 sec 2024-12-09T01:41:15,662 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.3250 sec 2024-12-09T01:41:15,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T01:41:15,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:15,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:15,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:15,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:15,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:15,747 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:15,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:15,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/b3ba5618bac6482ca124d64b65a74704 is 50, key is test_row_0/A:col10/1733708475103/Put/seqid=0 2024-12-09T01:41:15,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741942_1118 (size=14741) 2024-12-09T01:41:15,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708475800, max timestamp=1733708475879Num uniques 859; 2024-12-09T01:41:15,882 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708475800, max timestamp=1733708475879Num uniques 859; 2024-12-09T01:41:15,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:41:15,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:41:15,885 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708475800, max timestamp=1733708475879Num uniques 859; 2024-12-09T01:41:15,886 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:41:15,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:15,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708535961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:15,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:15,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708535969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:15,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:15,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:15,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:16,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:16,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708536075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:16,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:16,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:16,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708536083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:16,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/b3ba5618bac6482ca124d64b65a74704 2024-12-09T01:41:16,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:16,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34736 deadline: 1733708536215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:16,220 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_2' in 88980a4de80c5830889346818652c3a7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:41:16,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:16,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:16,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:16,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/7e1ef240700b413b9571ac9e7b40c7f6 is 50, key is test_row_0/B:col10/1733708475103/Put/seqid=0 2024-12-09T01:41:16,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34796 deadline: 1733708536244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:16,249 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_0' in 88980a4de80c5830889346818652c3a7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:41:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:16,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741943_1119 (size=12301) 2024-12-09T01:41:16,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/7e1ef240700b413b9571ac9e7b40c7f6 2024-12-09T01:41:16,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:16,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708536283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:16,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:16,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/79a7cb563d2b41eb95fa0054b7d49fe3 is 50, key is test_row_0/C:col10/1733708475103/Put/seqid=0 2024-12-09T01:41:16,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:16,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708536303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:16,311 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:16,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741944_1120 (size=12301) 2024-12-09T01:41:16,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/79a7cb563d2b41eb95fa0054b7d49fe3 2024-12-09T01:41:16,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/b3ba5618bac6482ca124d64b65a74704 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/b3ba5618bac6482ca124d64b65a74704 2024-12-09T01:41:16,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/b3ba5618bac6482ca124d64b65a74704, entries=200, sequenceid=560, filesize=14.4 K 2024-12-09T01:41:16,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/7e1ef240700b413b9571ac9e7b40c7f6 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/7e1ef240700b413b9571ac9e7b40c7f6 2024-12-09T01:41:16,369 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/7e1ef240700b413b9571ac9e7b40c7f6, entries=150, sequenceid=560, filesize=12.0 K 2024-12-09T01:41:16,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/79a7cb563d2b41eb95fa0054b7d49fe3 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/79a7cb563d2b41eb95fa0054b7d49fe3 2024-12-09T01:41:16,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/79a7cb563d2b41eb95fa0054b7d49fe3, entries=150, sequenceid=560, filesize=12.0 K 2024-12-09T01:41:16,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.94 KB/135110 for 88980a4de80c5830889346818652c3a7 in 634ms, sequenceid=560, compaction requested=true 2024-12-09T01:41:16,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:16,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:41:16,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:16,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:41:16,380 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:16,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:16,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:41:16,380 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:16,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:16,382 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40467 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:16,382 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/A is initiating minor compaction (all files) 2024-12-09T01:41:16,382 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/A in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:16,382 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/4059d70cab9247e791654c309b73d797, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ceb54252e8e149dd88b5bb9fb72c7e9c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/b3ba5618bac6482ca124d64b65a74704] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=39.5 K 2024-12-09T01:41:16,382 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:16,382 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/B is initiating minor compaction (all files) 2024-12-09T01:41:16,382 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/B in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:16,383 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4059d70cab9247e791654c309b73d797, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1733708474193 2024-12-09T01:41:16,383 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/c512c634069e464a999a22544dd59da6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/3acf36f60a034bc2909a6368567b2fd6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/7e1ef240700b413b9571ac9e7b40c7f6] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=37.1 K 2024-12-09T01:41:16,383 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting ceb54252e8e149dd88b5bb9fb72c7e9c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1733708474406 2024-12-09T01:41:16,383 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting c512c634069e464a999a22544dd59da6, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1733708474193 2024-12-09T01:41:16,383 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3ba5618bac6482ca124d64b65a74704, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=560, earliestPutTs=1733708475073 2024-12-09T01:41:16,384 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 3acf36f60a034bc2909a6368567b2fd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1733708474406 2024-12-09T01:41:16,384 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e1ef240700b413b9571ac9e7b40c7f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=560, earliestPutTs=1733708475093 2024-12-09T01:41:16,403 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#A#compaction#108 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:16,403 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/161151347594452099ab2b168cb57eb5 is 50, key is test_row_0/A:col10/1733708475103/Put/seqid=0 2024-12-09T01:41:16,411 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#B#compaction#109 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:16,412 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/1c65958d076940299ae7497b9e1a39e9 is 50, key is test_row_0/B:col10/1733708475103/Put/seqid=0 2024-12-09T01:41:16,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741945_1121 (size=13527) 2024-12-09T01:41:16,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741946_1122 (size=13527) 2024-12-09T01:41:16,479 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/1c65958d076940299ae7497b9e1a39e9 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/1c65958d076940299ae7497b9e1a39e9 2024-12-09T01:41:16,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-12-09T01:41:16,483 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:41:16,484 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:41:16,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-12-09T01:41:16,486 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:41:16,488 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:41:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T01:41:16,490 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:41:16,498 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/B of 88980a4de80c5830889346818652c3a7 into 1c65958d076940299ae7497b9e1a39e9(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:16,498 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:16,498 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/B, priority=13, startTime=1733708476380; duration=0sec 2024-12-09T01:41:16,499 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:16,499 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:B 2024-12-09T01:41:16,499 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:16,502 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:16,502 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/C is initiating minor compaction (all files) 2024-12-09T01:41:16,502 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/C in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:16,502 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/bc5f53f6935a4f918f7c05b8ee7d0e78, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/23ee3674e0194a138911397a9ec6f257, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/79a7cb563d2b41eb95fa0054b7d49fe3] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=37.1 K 2024-12-09T01:41:16,503 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting bc5f53f6935a4f918f7c05b8ee7d0e78, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=509, earliestPutTs=1733708474193 2024-12-09T01:41:16,503 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 23ee3674e0194a138911397a9ec6f257, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=548, earliestPutTs=1733708474406 2024-12-09T01:41:16,504 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 79a7cb563d2b41eb95fa0054b7d49fe3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=560, earliestPutTs=1733708475093 2024-12-09T01:41:16,525 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#C#compaction#110 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:16,526 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/a9b553ddc1f24726b6d3eb54506e8bdf is 50, key is test_row_0/C:col10/1733708475103/Put/seqid=0 2024-12-09T01:41:16,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741947_1123 (size=13527) 2024-12-09T01:41:16,556 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/a9b553ddc1f24726b6d3eb54506e8bdf as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/a9b553ddc1f24726b6d3eb54506e8bdf 2024-12-09T01:41:16,570 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/C of 88980a4de80c5830889346818652c3a7 into a9b553ddc1f24726b6d3eb54506e8bdf(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:16,570 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:16,570 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/C, priority=13, startTime=1733708476380; duration=0sec 2024-12-09T01:41:16,572 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:16,572 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:C 2024-12-09T01:41:16,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T01:41:16,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=138.65 KB heapSize=390.45 KB 2024-12-09T01:41:16,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:16,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:16,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:16,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:16,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:16,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:16,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:16,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T01:41:16,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:16,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:16,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:16,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:16,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:16,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/4e132b20f6e94cc0b465431e0615649c is 50, key is test_row_0/A:col10/1733708476604/Put/seqid=0 2024-12-09T01:41:16,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:16,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708536643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:16,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708536646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:16,656 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:16,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741948_1124 (size=12301) 2024-12-09T01:41:16,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/4e132b20f6e94cc0b465431e0615649c 2024-12-09T01:41:16,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/a9c635cb49fc496b9dfabf6f7a60ddfa is 50, key is test_row_0/B:col10/1733708476604/Put/seqid=0 2024-12-09T01:41:16,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741949_1125 (size=12301) 2024-12-09T01:41:16,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/a9c635cb49fc496b9dfabf6f7a60ddfa 2024-12-09T01:41:16,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/5364d1740f994ea28ea0e7921168ded0 is 50, key is test_row_0/C:col10/1733708476604/Put/seqid=0 2024-12-09T01:41:16,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741950_1126 (size=12301) 2024-12-09T01:41:16,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=598 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/5364d1740f994ea28ea0e7921168ded0 2024-12-09T01:41:16,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/4e132b20f6e94cc0b465431e0615649c as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/4e132b20f6e94cc0b465431e0615649c 2024-12-09T01:41:16,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:16,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708536768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:16,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:16,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:16,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:16,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:16,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708536768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:16,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:16,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:16,772 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:16,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/4e132b20f6e94cc0b465431e0615649c, entries=150, sequenceid=598, filesize=12.0 K 2024-12-09T01:41:16,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/a9c635cb49fc496b9dfabf6f7a60ddfa as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/a9c635cb49fc496b9dfabf6f7a60ddfa 2024-12-09T01:41:16,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T01:41:16,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:16,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:16,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:16,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:16,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:16,800 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/a9c635cb49fc496b9dfabf6f7a60ddfa, entries=150, sequenceid=598, filesize=12.0 K 2024-12-09T01:41:16,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:16,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/5364d1740f994ea28ea0e7921168ded0 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/5364d1740f994ea28ea0e7921168ded0 2024-12-09T01:41:16,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T01:41:16,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/5364d1740f994ea28ea0e7921168ded0, entries=150, sequenceid=598, filesize=12.0 K 2024-12-09T01:41:16,815 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~138.65 KB/141980, heapSize ~390.41 KB/399776, currentSize=46.96 KB/48090 for 88980a4de80c5830889346818652c3a7 in 209ms, sequenceid=598, compaction requested=false 2024-12-09T01:41:16,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:16,865 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/161151347594452099ab2b168cb57eb5 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/161151347594452099ab2b168cb57eb5 2024-12-09T01:41:16,874 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/A of 88980a4de80c5830889346818652c3a7 into 161151347594452099ab2b168cb57eb5(size=13.2 K), total size for store is 25.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:16,875 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:16,875 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/A, priority=13, startTime=1733708476380; duration=0sec 2024-12-09T01:41:16,875 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:16,875 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:A 2024-12-09T01:41:16,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-12-09T01:41:16,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:16,953 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:41:16,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:16,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:16,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:16,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:16,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:16,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:16,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/1cdd4c8d34634d9c85408b0eb303b39e is 50, key is test_row_0/A:col10/1733708476643/Put/seqid=0 2024-12-09T01:41:16,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:16,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:16,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741951_1127 (size=12301) 2024-12-09T01:41:16,989 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=609 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/1cdd4c8d34634d9c85408b0eb303b39e 2024-12-09T01:41:17,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/488e48e9d5df4c9f82fcc172fd4f64f1 is 50, key is test_row_0/B:col10/1733708476643/Put/seqid=0 2024-12-09T01:41:17,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741952_1128 (size=12301) 2024-12-09T01:41:17,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T01:41:17,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708477063, max timestamp=1733708477117Num uniques 859; 2024-12-09T01:41:17,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:41:17,125 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708477063, max timestamp=1733708477118Num uniques 859; 2024-12-09T01:41:17,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:41:17,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708477063, max timestamp=1733708477117Num uniques 859; 2024-12-09T01:41:17,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:41:17,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:17,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708537217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:17,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:17,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708537220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:17,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:17,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:17,226 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:17,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:17,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:17,226 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:17,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:17,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34722 deadline: 1733708537273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:17,278 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_0' in 88980a4de80c5830889346818652c3a7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8226 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:41:17,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:17,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:17,279 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:17,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:17,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708537333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:17,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:17,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708537334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:17,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:17,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:17,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:17,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:17,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:17,339 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:17,456 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=609 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/488e48e9d5df4c9f82fcc172fd4f64f1 2024-12-09T01:41:17,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/941d5d375bed44d78403b123cc8832a0 is 50, key is test_row_0/C:col10/1733708476643/Put/seqid=0 2024-12-09T01:41:17,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741953_1129 (size=12301) 2024-12-09T01:41:17,484 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=609 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/941d5d375bed44d78403b123cc8832a0 2024-12-09T01:41:17,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/1cdd4c8d34634d9c85408b0eb303b39e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/1cdd4c8d34634d9c85408b0eb303b39e 2024-12-09T01:41:17,499 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/1cdd4c8d34634d9c85408b0eb303b39e, entries=150, sequenceid=609, filesize=12.0 K 2024-12-09T01:41:17,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/488e48e9d5df4c9f82fcc172fd4f64f1 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/488e48e9d5df4c9f82fcc172fd4f64f1 2024-12-09T01:41:17,508 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/488e48e9d5df4c9f82fcc172fd4f64f1, entries=150, sequenceid=609, filesize=12.0 K 2024-12-09T01:41:17,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/941d5d375bed44d78403b123cc8832a0 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/941d5d375bed44d78403b123cc8832a0 2024-12-09T01:41:17,516 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/941d5d375bed44d78403b123cc8832a0, entries=150, sequenceid=609, filesize=12.0 K 2024-12-09T01:41:17,518 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 88980a4de80c5830889346818652c3a7 in 565ms, sequenceid=609, compaction requested=true 2024-12-09T01:41:17,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:17,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:17,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-09T01:41:17,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-09T01:41:17,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-09T01:41:17,523 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0330 sec 2024-12-09T01:41:17,525 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.0400 sec 2024-12-09T01:41:17,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:17,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733708477189, max timestamp=1733708477553Num uniques 1018; 2024-12-09T01:41:17,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T01:41:17,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733708477189, max timestamp=1733708477553Num uniques 1018; 2024-12-09T01:41:17,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T01:41:17,560 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=64.85 KB heapSize=178.80 KB 2024-12-09T01:41:17,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:17,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:17,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:17,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:17,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:17,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T01:41:17,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/991fea3917f040a8a523d8434b01dec5 is 50, key is test_row_0/A:col10/1733708477553/Put/seqid=0 2024-12-09T01:41:17,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-09T01:41:17,625 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:41:17,625 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:41:17,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-12-09T01:41:17,628 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:41:17,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T01:41:17,629 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:41:17,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:41:17,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741954_1130 (size=14741) 2024-12-09T01:41:17,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708537669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:17,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708537669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:17,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T01:41:17,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-09T01:41:17,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:17,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:17,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:17,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:17,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:17,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:17,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708537793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:17,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:17,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:17,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708537793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:17,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:17,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:17,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:17,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-09T01:41:17,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:17,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:17,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:17,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:17,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:17,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T01:41:18,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708538013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:18,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:18,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708538013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:18,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:18,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:18,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:18,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:18,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:18,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=648 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/991fea3917f040a8a523d8434b01dec5 2024-12-09T01:41:18,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/f4e37893eab94184b9e6aab16de1ce69 is 50, key is test_row_0/B:col10/1733708477553/Put/seqid=0 2024-12-09T01:41:18,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-09T01:41:18,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:18,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:18,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:18,092 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:18,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:18,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:18,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741955_1131 (size=12301) 2024-12-09T01:41:18,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=648 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/f4e37893eab94184b9e6aab16de1ce69 2024-12-09T01:41:18,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/1ce3cbb25fcc4885bdc1264b7a19f88b is 50, key is test_row_0/C:col10/1733708477553/Put/seqid=0 2024-12-09T01:41:18,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741956_1132 (size=12301) 2024-12-09T01:41:18,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=648 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/1ce3cbb25fcc4885bdc1264b7a19f88b 2024-12-09T01:41:18,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/991fea3917f040a8a523d8434b01dec5 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/991fea3917f040a8a523d8434b01dec5 2024-12-09T01:41:18,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/991fea3917f040a8a523d8434b01dec5, entries=200, sequenceid=648, filesize=14.4 K 2024-12-09T01:41:18,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/f4e37893eab94184b9e6aab16de1ce69 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/f4e37893eab94184b9e6aab16de1ce69 2024-12-09T01:41:18,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/f4e37893eab94184b9e6aab16de1ce69, entries=150, sequenceid=648, filesize=12.0 K 2024-12-09T01:41:18,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/1ce3cbb25fcc4885bdc1264b7a19f88b as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/1ce3cbb25fcc4885bdc1264b7a19f88b 2024-12-09T01:41:18,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/1ce3cbb25fcc4885bdc1264b7a19f88b, entries=150, sequenceid=648, filesize=12.0 K 2024-12-09T01:41:18,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~198.67 KB/203440, currentSize=114.05 KB/116790 for 88980a4de80c5830889346818652c3a7 in 684ms, sequenceid=648, compaction requested=true 2024-12-09T01:41:18,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:18,245 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:41:18,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-12-09T01:41:18,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:18,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:41:18,246 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T01:41:18,246 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52870 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:41:18,247 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/A is initiating minor compaction (all files) 2024-12-09T01:41:18,247 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/A in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:18,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:18,247 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/161151347594452099ab2b168cb57eb5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/4e132b20f6e94cc0b465431e0615649c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/1cdd4c8d34634d9c85408b0eb303b39e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/991fea3917f040a8a523d8434b01dec5] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=51.6 K 2024-12-09T01:41:18,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:18,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:18,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:18,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:18,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:18,248 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 161151347594452099ab2b168cb57eb5, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=560, earliestPutTs=1733708475093 2024-12-09T01:41:18,249 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e132b20f6e94cc0b465431e0615649c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=598, earliestPutTs=1733708475944 2024-12-09T01:41:18,249 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cdd4c8d34634d9c85408b0eb303b39e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=609, earliestPutTs=1733708476625 2024-12-09T01:41:18,253 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 991fea3917f040a8a523d8434b01dec5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=648, earliestPutTs=1733708477189 2024-12-09T01:41:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T01:41:18,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:18,255 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:41:18,257 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50430 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:41:18,257 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/B is initiating minor compaction (all files) 2024-12-09T01:41:18,257 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/B in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:18,257 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/1c65958d076940299ae7497b9e1a39e9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/a9c635cb49fc496b9dfabf6f7a60ddfa, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/488e48e9d5df4c9f82fcc172fd4f64f1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/f4e37893eab94184b9e6aab16de1ce69] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=49.2 K 2024-12-09T01:41:18,258 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c65958d076940299ae7497b9e1a39e9, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=560, earliestPutTs=1733708475093 2024-12-09T01:41:18,259 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting a9c635cb49fc496b9dfabf6f7a60ddfa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=598, earliestPutTs=1733708475944 2024-12-09T01:41:18,259 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 488e48e9d5df4c9f82fcc172fd4f64f1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=609, earliestPutTs=1733708476625 2024-12-09T01:41:18,260 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting f4e37893eab94184b9e6aab16de1ce69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=648, earliestPutTs=1733708477189 2024-12-09T01:41:18,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:41:18,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:18,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:41:18,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:18,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/ef43acbfaa154aab88f4a1ff03a43a4f is 50, key is test_row_0/A:col10/1733708477666/Put/seqid=0 2024-12-09T01:41:18,276 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#A#compaction#121 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:18,277 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/66e5372138e84908a1b7fe1f662369c2 is 50, key is test_row_0/A:col10/1733708477553/Put/seqid=0 2024-12-09T01:41:18,283 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#B#compaction#122 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:18,283 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/e3d2a73594d5498da8684e17d968c08e is 50, key is test_row_0/B:col10/1733708477553/Put/seqid=0 2024-12-09T01:41:18,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741958_1134 (size=13663) 2024-12-09T01:41:18,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:18,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. as already flushing 2024-12-09T01:41:18,334 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/66e5372138e84908a1b7fe1f662369c2 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/66e5372138e84908a1b7fe1f662369c2 2024-12-09T01:41:18,346 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88980a4de80c5830889346818652c3a7/A of 88980a4de80c5830889346818652c3a7 into 66e5372138e84908a1b7fe1f662369c2(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:18,346 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:18,346 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/A, priority=12, startTime=1733708478244; duration=0sec 2024-12-09T01:41:18,346 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:18,346 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:A 2024-12-09T01:41:18,346 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:41:18,349 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50430 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:41:18,349 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/C is initiating minor compaction (all files) 2024-12-09T01:41:18,349 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/C in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:18,349 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/a9b553ddc1f24726b6d3eb54506e8bdf, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/5364d1740f994ea28ea0e7921168ded0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/941d5d375bed44d78403b123cc8832a0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/1ce3cbb25fcc4885bdc1264b7a19f88b] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=49.2 K 2024-12-09T01:41:18,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741959_1135 (size=13663) 2024-12-09T01:41:18,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741957_1133 (size=12301) 2024-12-09T01:41:18,353 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9b553ddc1f24726b6d3eb54506e8bdf, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=560, earliestPutTs=1733708475093 2024-12-09T01:41:18,354 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5364d1740f994ea28ea0e7921168ded0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=598, earliestPutTs=1733708475944 2024-12-09T01:41:18,358 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/e3d2a73594d5498da8684e17d968c08e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/e3d2a73594d5498da8684e17d968c08e 2024-12-09T01:41:18,363 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 941d5d375bed44d78403b123cc8832a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=609, earliestPutTs=1733708476625 2024-12-09T01:41:18,366 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ce3cbb25fcc4885bdc1264b7a19f88b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=648, earliestPutTs=1733708477189 2024-12-09T01:41:18,376 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88980a4de80c5830889346818652c3a7/B of 88980a4de80c5830889346818652c3a7 into e3d2a73594d5498da8684e17d968c08e(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:18,376 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:18,376 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/B, priority=12, startTime=1733708478255; duration=0sec 2024-12-09T01:41:18,377 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:18,377 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:B 2024-12-09T01:41:18,382 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#C#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:18,383 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/ec1060b1c4c849298599f46b5eebc5cd is 50, key is test_row_0/C:col10/1733708477553/Put/seqid=0 2024-12-09T01:41:18,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:18,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708538399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:18,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:18,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:18,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:18,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:18,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708538415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:18,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:18,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741960_1136 (size=13663) 2024-12-09T01:41:18,467 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/ec1060b1c4c849298599f46b5eebc5cd as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/ec1060b1c4c849298599f46b5eebc5cd 2024-12-09T01:41:18,475 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88980a4de80c5830889346818652c3a7/C of 88980a4de80c5830889346818652c3a7 into ec1060b1c4c849298599f46b5eebc5cd(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:18,476 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:18,476 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/C, priority=12, startTime=1733708478260; duration=0sec 2024-12-09T01:41:18,476 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:18,476 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:C 2024-12-09T01:41:18,536 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:18,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708538524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:18,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:18,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708538524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:18,537 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:18,538 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:18,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:18,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708538743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:18,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:18,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:18,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:18,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708538743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:18,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:18,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:18,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:18,753 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=668 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/ef43acbfaa154aab88f4a1ff03a43a4f 2024-12-09T01:41:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T01:41:18,769 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-12-09T01:41:18,769 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:41:18,772 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-12-09T01:41:18,772 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:41:18,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/23a0fd5b7d5c4e83a6dbf895c05a0c85 is 50, key is test_row_0/B:col10/1733708477666/Put/seqid=0 2024-12-09T01:41:18,773 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-12-09T01:41:18,773 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:41:18,775 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-12-09T01:41:18,775 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:41:18,776 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:18,776 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:18,776 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:18,776 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:18,776 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:18,776 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:18,776 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:41:18,776 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:18,776 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:41:18,776 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:18,776 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:41:18,776 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:41:18,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741961_1137 (size=12301) 2024-12-09T01:41:19,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 251 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34786 deadline: 1733708539053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:19,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:19,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:19,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:19,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:19,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34760 deadline: 1733708539063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:19,064 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:41:19,180 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=668 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/23a0fd5b7d5c4e83a6dbf895c05a0c85 2024-12-09T01:41:19,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/536444b497e34db4ab5860425b00a3f0 is 50, key is test_row_0/C:col10/1733708477666/Put/seqid=0 2024-12-09T01:41:19,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741962_1138 (size=12301) 2024-12-09T01:41:19,194 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=668 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/536444b497e34db4ab5860425b00a3f0 2024-12-09T01:41:19,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/ef43acbfaa154aab88f4a1ff03a43a4f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ef43acbfaa154aab88f4a1ff03a43a4f 2024-12-09T01:41:19,204 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ef43acbfaa154aab88f4a1ff03a43a4f, entries=150, sequenceid=668, filesize=12.0 K 2024-12-09T01:41:19,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/23a0fd5b7d5c4e83a6dbf895c05a0c85 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/23a0fd5b7d5c4e83a6dbf895c05a0c85 2024-12-09T01:41:19,209 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/23a0fd5b7d5c4e83a6dbf895c05a0c85, entries=150, sequenceid=668, filesize=12.0 K 2024-12-09T01:41:19,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/536444b497e34db4ab5860425b00a3f0 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/536444b497e34db4ab5860425b00a3f0 2024-12-09T01:41:19,214 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/536444b497e34db4ab5860425b00a3f0, entries=150, sequenceid=668, filesize=12.0 K 2024-12-09T01:41:19,215 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 88980a4de80c5830889346818652c3a7 in 969ms, sequenceid=668, compaction requested=false 2024-12-09T01:41:19,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:19,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:19,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-12-09T01:41:19,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-12-09T01:41:19,218 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-09T01:41:19,218 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5880 sec 2024-12-09T01:41:19,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.5940 sec 2024-12-09T01:41:19,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:19,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T01:41:19,565 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-12-09T01:41:19,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:19,565 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:41:19,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:19,565 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:19,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:19,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:19,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:19,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:19,565 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:19,565 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:41:19,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/88cacb946e2548abb2bed9357eaadf71 is 50, key is test_row_0/A:col10/1733708479563/Put/seqid=0 2024-12-09T01:41:19,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741963_1139 (size=12301) 2024-12-09T01:41:19,574 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-12-09T01:41:19,575 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:41:19,575 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:19,575 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:19,575 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:41:19,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-09T01:41:19,774 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:41:19,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=684 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/88cacb946e2548abb2bed9357eaadf71 2024-12-09T01:41:19,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/f14ef63966e84da38f2b81050b204725 is 50, key is test_row_0/B:col10/1733708479563/Put/seqid=0 2024-12-09T01:41:19,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741964_1140 (size=12301) 2024-12-09T01:41:20,387 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=684 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/f14ef63966e84da38f2b81050b204725 2024-12-09T01:41:20,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/57e8fce0bb73413a9be8718bdcc08fac is 50, key is test_row_0/C:col10/1733708479563/Put/seqid=0 2024-12-09T01:41:20,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741965_1141 (size=12301) 2024-12-09T01:41:20,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=684 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/57e8fce0bb73413a9be8718bdcc08fac 2024-12-09T01:41:20,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/88cacb946e2548abb2bed9357eaadf71 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/88cacb946e2548abb2bed9357eaadf71 2024-12-09T01:41:20,808 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/88cacb946e2548abb2bed9357eaadf71, entries=150, sequenceid=684, filesize=12.0 K 2024-12-09T01:41:20,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/f14ef63966e84da38f2b81050b204725 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/f14ef63966e84da38f2b81050b204725 2024-12-09T01:41:20,814 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/f14ef63966e84da38f2b81050b204725, entries=150, sequenceid=684, filesize=12.0 K 2024-12-09T01:41:20,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/57e8fce0bb73413a9be8718bdcc08fac as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/57e8fce0bb73413a9be8718bdcc08fac 2024-12-09T01:41:20,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/57e8fce0bb73413a9be8718bdcc08fac, entries=150, sequenceid=684, filesize=12.0 K 2024-12-09T01:41:20,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=6.71 KB/6870 for 88980a4de80c5830889346818652c3a7 in 1256ms, sequenceid=684, compaction requested=true 2024-12-09T01:41:20,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:20,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:41:20,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:20,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:41:20,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:20,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88980a4de80c5830889346818652c3a7:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:41:20,820 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:20,820 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:20,820 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:20,821 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:20,821 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:20,821 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/B is initiating minor compaction (all files) 2024-12-09T01:41:20,821 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/A is initiating minor compaction (all files) 2024-12-09T01:41:20,821 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/B in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:20,821 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/A in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:20,821 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/e3d2a73594d5498da8684e17d968c08e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/23a0fd5b7d5c4e83a6dbf895c05a0c85, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/f14ef63966e84da38f2b81050b204725] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=37.4 K 2024-12-09T01:41:20,821 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/66e5372138e84908a1b7fe1f662369c2, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ef43acbfaa154aab88f4a1ff03a43a4f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/88cacb946e2548abb2bed9357eaadf71] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=37.4 K 2024-12-09T01:41:20,822 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting e3d2a73594d5498da8684e17d968c08e, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=648, earliestPutTs=1733708477189 2024-12-09T01:41:20,822 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66e5372138e84908a1b7fe1f662369c2, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=648, earliestPutTs=1733708477189 2024-12-09T01:41:20,822 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 23a0fd5b7d5c4e83a6dbf895c05a0c85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=668, earliestPutTs=1733708477653 2024-12-09T01:41:20,822 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef43acbfaa154aab88f4a1ff03a43a4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=668, earliestPutTs=1733708477653 2024-12-09T01:41:20,822 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting f14ef63966e84da38f2b81050b204725, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=684, earliestPutTs=1733708478396 2024-12-09T01:41:20,823 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88cacb946e2548abb2bed9357eaadf71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=684, earliestPutTs=1733708478396 2024-12-09T01:41:20,831 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#B#compaction#129 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:20,832 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/6b27999909024f14b29bd880b3725c7f is 50, key is test_row_0/B:col10/1733708479563/Put/seqid=0 2024-12-09T01:41:20,832 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#A#compaction#130 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:20,833 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/63c46457ffbd473685a0db855c0c9caa is 50, key is test_row_0/A:col10/1733708479563/Put/seqid=0 2024-12-09T01:41:20,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741966_1142 (size=13765) 2024-12-09T01:41:20,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741967_1143 (size=13765) 2024-12-09T01:41:20,842 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/63c46457ffbd473685a0db855c0c9caa as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/63c46457ffbd473685a0db855c0c9caa 2024-12-09T01:41:20,848 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/A of 88980a4de80c5830889346818652c3a7 into 63c46457ffbd473685a0db855c0c9caa(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:20,848 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:20,848 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/A, priority=13, startTime=1733708480820; duration=0sec 2024-12-09T01:41:20,848 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:20,848 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:A 2024-12-09T01:41:20,848 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:20,849 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:20,850 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 88980a4de80c5830889346818652c3a7/C is initiating minor compaction (all files) 2024-12-09T01:41:20,850 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88980a4de80c5830889346818652c3a7/C in TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:20,850 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/ec1060b1c4c849298599f46b5eebc5cd, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/536444b497e34db4ab5860425b00a3f0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/57e8fce0bb73413a9be8718bdcc08fac] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp, totalSize=37.4 K 2024-12-09T01:41:20,850 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec1060b1c4c849298599f46b5eebc5cd, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=648, earliestPutTs=1733708477189 2024-12-09T01:41:20,851 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 536444b497e34db4ab5860425b00a3f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=668, earliestPutTs=1733708477653 2024-12-09T01:41:20,851 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57e8fce0bb73413a9be8718bdcc08fac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=684, earliestPutTs=1733708478396 2024-12-09T01:41:20,858 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88980a4de80c5830889346818652c3a7#C#compaction#131 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:20,858 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/98157620e15b4fd0be19440a0b3f16f6 is 50, key is test_row_0/C:col10/1733708479563/Put/seqid=0 2024-12-09T01:41:20,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741968_1144 (size=13765) 2024-12-09T01:41:21,242 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/6b27999909024f14b29bd880b3725c7f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/6b27999909024f14b29bd880b3725c7f 2024-12-09T01:41:21,247 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/B of 88980a4de80c5830889346818652c3a7 into 6b27999909024f14b29bd880b3725c7f(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:21,247 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:21,247 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/B, priority=13, startTime=1733708480820; duration=0sec 2024-12-09T01:41:21,247 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:21,247 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:B 2024-12-09T01:41:21,266 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/98157620e15b4fd0be19440a0b3f16f6 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/98157620e15b4fd0be19440a0b3f16f6 2024-12-09T01:41:21,271 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88980a4de80c5830889346818652c3a7/C of 88980a4de80c5830889346818652c3a7 into 98157620e15b4fd0be19440a0b3f16f6(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:21,271 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88980a4de80c5830889346818652c3a7: 2024-12-09T01:41:21,271 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., storeName=88980a4de80c5830889346818652c3a7/C, priority=13, startTime=1733708480820; duration=0sec 2024-12-09T01:41:21,271 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:21,271 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88980a4de80c5830889346818652c3a7:C 2024-12-09T01:41:21,957 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T01:41:26,266 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-12-09T01:41:26,266 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:41:26,266 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:26,266 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:26,266 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:41:26,356 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-12-09T01:41:26,356 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:41:26,356 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:26,356 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:26,356 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:41:27,335 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-12-09T01:41:27,335 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:41:27,335 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:27,335 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:27,335 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:41:27,335 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T01:41:27,336 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 188 2024-12-09T01:41:27,336 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-12-09T01:41:27,336 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 181 2024-12-09T01:41:27,336 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-12-09T01:41:27,336 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-12-09T01:41:27,336 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T01:41:27,336 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2187 2024-12-09T01:41:27,336 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2303 2024-12-09T01:41:27,336 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T01:41:27,336 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1040 2024-12-09T01:41:27,336 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3112 rows 2024-12-09T01:41:27,336 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1037 2024-12-09T01:41:27,336 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3102 rows 2024-12-09T01:41:27,336 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T01:41:27,336 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T01:41:27,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:27,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:27,336 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:41:27,341 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T01:41:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T01:41:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T01:41:27,349 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708487349"}]},"ts":"1733708487349"} 2024-12-09T01:41:27,351 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T01:41:27,351 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T01:41:27,353 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T01:41:27,358 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88980a4de80c5830889346818652c3a7, UNASSIGN}] 2024-12-09T01:41:27,359 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88980a4de80c5830889346818652c3a7, UNASSIGN 2024-12-09T01:41:27,360 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=88980a4de80c5830889346818652c3a7, regionState=CLOSING, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:27,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=88980a4de80c5830889346818652c3a7, UNASSIGN because future has completed 2024-12-09T01:41:27,363 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T01:41:27,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943}] 2024-12-09T01:41:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T01:41:27,520 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:27,521 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T01:41:27,521 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 88980a4de80c5830889346818652c3a7, disabling compactions & flushes 2024-12-09T01:41:27,521 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:27,521 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:27,521 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. after waiting 0 ms 2024-12-09T01:41:27,521 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:27,521 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 88980a4de80c5830889346818652c3a7 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-09T01:41:27,522 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=A 2024-12-09T01:41:27,522 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:27,522 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=B 2024-12-09T01:41:27,522 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:27,522 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 88980a4de80c5830889346818652c3a7, store=C 2024-12-09T01:41:27,522 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:27,529 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/ae21897c0cbd48cd963f228f3a9004d7 is 50, key is test_row_0/A:col10/1733708487333/Put/seqid=0 2024-12-09T01:41:27,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741969_1145 (size=9857) 2024-12-09T01:41:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T01:41:27,935 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=694 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/ae21897c0cbd48cd963f228f3a9004d7 2024-12-09T01:41:27,946 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/c7ecc089ae72441e82ed2a514de59fdf is 50, key is test_row_0/B:col10/1733708487333/Put/seqid=0 2024-12-09T01:41:27,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741970_1146 (size=9857) 2024-12-09T01:41:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T01:41:28,352 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=694 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/c7ecc089ae72441e82ed2a514de59fdf 2024-12-09T01:41:28,363 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/c524ca0a24564060af561e35719f2b23 is 50, key is test_row_0/C:col10/1733708487333/Put/seqid=0 2024-12-09T01:41:28,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741971_1147 (size=9857) 2024-12-09T01:41:28,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T01:41:28,779 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=694 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/c524ca0a24564060af561e35719f2b23 2024-12-09T01:41:28,794 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/A/ae21897c0cbd48cd963f228f3a9004d7 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ae21897c0cbd48cd963f228f3a9004d7 2024-12-09T01:41:28,799 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ae21897c0cbd48cd963f228f3a9004d7, entries=100, sequenceid=694, filesize=9.6 K 2024-12-09T01:41:28,800 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/B/c7ecc089ae72441e82ed2a514de59fdf as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/c7ecc089ae72441e82ed2a514de59fdf 2024-12-09T01:41:28,807 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/c7ecc089ae72441e82ed2a514de59fdf, entries=100, sequenceid=694, filesize=9.6 K 2024-12-09T01:41:28,808 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/.tmp/C/c524ca0a24564060af561e35719f2b23 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/c524ca0a24564060af561e35719f2b23 2024-12-09T01:41:28,814 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/c524ca0a24564060af561e35719f2b23, entries=100, sequenceid=694, filesize=9.6 K 2024-12-09T01:41:28,815 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 88980a4de80c5830889346818652c3a7 in 1294ms, sequenceid=694, compaction requested=false 2024-12-09T01:41:28,816 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/e5fe0ca26d454dad92cb4167e14f4a35, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/c4f4e71da53b4be3a8538e1c9389a283, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/37b12b31e923443db3bef1bfd3478581, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ef564974680248f0a429d59e0d430cba, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/9e344e5757c2476fbc617b851fabc862, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/8006b6bbf9904dd18d58061214268fd0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/3cce67efe6ab418083eb6fb80635a474, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/7b4c717295f6420f9466acf91028d833, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/89f7b99f4bd8402687bf64cfaa87e6a3, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/45fe309f77774ceebe76fb01f4dc554f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/f8d951a721c645e7ba0f4a36af10fe1a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/df8ae9e610b249cfbc5e12dae8635f0e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/01da971ef2024d15b090f5f6143e576a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ac30044b395a49068753ed0fc592ca0f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/3554977c95d24546806cae9740a46f17, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/700e2649495c43adad5b4248567d5ab1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/1022950a507d4fbe81a576bbe7a91bcd, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/9c7860baf5da452bb1d2293330403d79, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/a18c98172af7451b826ed629d0356c2e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/43fb489db2064920b061cc9615bae672, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/3f0c15cb75394ac4a25a4fe5a52e3980, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/47545c7762964bb2953ca20cddab98af, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/787853de78974f4b83e42e192fa0e01b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/8eb859a81cde4e61a78e5d4d96d18200, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/b902f4d2f75949dcae4a870d0f5aed64, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/825e790e83474419bf191e90a3ee9e47, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/05074c791b0e476987d44bfde5793996, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/4942f7209839421885f2e1c59a15fffc, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/3a9441b5dda54cc1bb8c5a0b176ea3c9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/796865ff55654e3a812d67bf6708d119, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/1950b0682a0949d3a111779b948657d4, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/b1f2ad8a58da4d248b8bed7d8dd0767c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/429ebb2636a048c689c27cc03b9463de, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/4059d70cab9247e791654c309b73d797, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ceb54252e8e149dd88b5bb9fb72c7e9c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/b3ba5618bac6482ca124d64b65a74704, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/161151347594452099ab2b168cb57eb5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/4e132b20f6e94cc0b465431e0615649c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/1cdd4c8d34634d9c85408b0eb303b39e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/991fea3917f040a8a523d8434b01dec5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/66e5372138e84908a1b7fe1f662369c2, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ef43acbfaa154aab88f4a1ff03a43a4f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/88cacb946e2548abb2bed9357eaadf71] to archive 2024-12-09T01:41:28,822 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T01:41:28,833 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/e5fe0ca26d454dad92cb4167e14f4a35 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/e5fe0ca26d454dad92cb4167e14f4a35 2024-12-09T01:41:28,835 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/c4f4e71da53b4be3a8538e1c9389a283 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/c4f4e71da53b4be3a8538e1c9389a283 2024-12-09T01:41:28,838 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/37b12b31e923443db3bef1bfd3478581 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/37b12b31e923443db3bef1bfd3478581 2024-12-09T01:41:28,839 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ef564974680248f0a429d59e0d430cba to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ef564974680248f0a429d59e0d430cba 2024-12-09T01:41:28,841 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/9e344e5757c2476fbc617b851fabc862 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/9e344e5757c2476fbc617b851fabc862 2024-12-09T01:41:28,845 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/8006b6bbf9904dd18d58061214268fd0 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/8006b6bbf9904dd18d58061214268fd0 2024-12-09T01:41:28,847 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/3cce67efe6ab418083eb6fb80635a474 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/3cce67efe6ab418083eb6fb80635a474 2024-12-09T01:41:28,848 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/7b4c717295f6420f9466acf91028d833 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/7b4c717295f6420f9466acf91028d833 2024-12-09T01:41:28,850 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/89f7b99f4bd8402687bf64cfaa87e6a3 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/89f7b99f4bd8402687bf64cfaa87e6a3 2024-12-09T01:41:28,852 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/45fe309f77774ceebe76fb01f4dc554f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/45fe309f77774ceebe76fb01f4dc554f 2024-12-09T01:41:28,853 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/f8d951a721c645e7ba0f4a36af10fe1a to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/f8d951a721c645e7ba0f4a36af10fe1a 2024-12-09T01:41:28,855 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/df8ae9e610b249cfbc5e12dae8635f0e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/df8ae9e610b249cfbc5e12dae8635f0e 2024-12-09T01:41:28,857 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/01da971ef2024d15b090f5f6143e576a to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/01da971ef2024d15b090f5f6143e576a 2024-12-09T01:41:28,859 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ac30044b395a49068753ed0fc592ca0f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ac30044b395a49068753ed0fc592ca0f 2024-12-09T01:41:28,860 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/3554977c95d24546806cae9740a46f17 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/3554977c95d24546806cae9740a46f17 2024-12-09T01:41:28,862 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/700e2649495c43adad5b4248567d5ab1 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/700e2649495c43adad5b4248567d5ab1 2024-12-09T01:41:28,864 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/1022950a507d4fbe81a576bbe7a91bcd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/1022950a507d4fbe81a576bbe7a91bcd 2024-12-09T01:41:28,866 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/9c7860baf5da452bb1d2293330403d79 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/9c7860baf5da452bb1d2293330403d79 2024-12-09T01:41:28,868 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/a18c98172af7451b826ed629d0356c2e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/a18c98172af7451b826ed629d0356c2e 2024-12-09T01:41:28,870 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/43fb489db2064920b061cc9615bae672 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/43fb489db2064920b061cc9615bae672 2024-12-09T01:41:28,871 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/3f0c15cb75394ac4a25a4fe5a52e3980 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/3f0c15cb75394ac4a25a4fe5a52e3980 2024-12-09T01:41:28,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/47545c7762964bb2953ca20cddab98af to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/47545c7762964bb2953ca20cddab98af 2024-12-09T01:41:28,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/787853de78974f4b83e42e192fa0e01b to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/787853de78974f4b83e42e192fa0e01b 2024-12-09T01:41:28,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/8eb859a81cde4e61a78e5d4d96d18200 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/8eb859a81cde4e61a78e5d4d96d18200 2024-12-09T01:41:28,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/b902f4d2f75949dcae4a870d0f5aed64 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/b902f4d2f75949dcae4a870d0f5aed64 2024-12-09T01:41:28,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/825e790e83474419bf191e90a3ee9e47 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/825e790e83474419bf191e90a3ee9e47 2024-12-09T01:41:28,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/05074c791b0e476987d44bfde5793996 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/05074c791b0e476987d44bfde5793996 2024-12-09T01:41:28,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/4942f7209839421885f2e1c59a15fffc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/4942f7209839421885f2e1c59a15fffc 2024-12-09T01:41:28,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/3a9441b5dda54cc1bb8c5a0b176ea3c9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/3a9441b5dda54cc1bb8c5a0b176ea3c9 2024-12-09T01:41:28,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/796865ff55654e3a812d67bf6708d119 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/796865ff55654e3a812d67bf6708d119 2024-12-09T01:41:28,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/1950b0682a0949d3a111779b948657d4 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/1950b0682a0949d3a111779b948657d4 2024-12-09T01:41:28,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/b1f2ad8a58da4d248b8bed7d8dd0767c to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/b1f2ad8a58da4d248b8bed7d8dd0767c 2024-12-09T01:41:28,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/429ebb2636a048c689c27cc03b9463de to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/429ebb2636a048c689c27cc03b9463de 2024-12-09T01:41:28,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/4059d70cab9247e791654c309b73d797 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/4059d70cab9247e791654c309b73d797 2024-12-09T01:41:28,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ceb54252e8e149dd88b5bb9fb72c7e9c to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ceb54252e8e149dd88b5bb9fb72c7e9c 2024-12-09T01:41:28,899 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/b3ba5618bac6482ca124d64b65a74704 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/b3ba5618bac6482ca124d64b65a74704 2024-12-09T01:41:28,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/161151347594452099ab2b168cb57eb5 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/161151347594452099ab2b168cb57eb5 2024-12-09T01:41:28,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/4e132b20f6e94cc0b465431e0615649c to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/4e132b20f6e94cc0b465431e0615649c 2024-12-09T01:41:28,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/1cdd4c8d34634d9c85408b0eb303b39e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/1cdd4c8d34634d9c85408b0eb303b39e 2024-12-09T01:41:28,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/991fea3917f040a8a523d8434b01dec5 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/991fea3917f040a8a523d8434b01dec5 2024-12-09T01:41:28,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/66e5372138e84908a1b7fe1f662369c2 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/66e5372138e84908a1b7fe1f662369c2 2024-12-09T01:41:28,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ef43acbfaa154aab88f4a1ff03a43a4f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ef43acbfaa154aab88f4a1ff03a43a4f 2024-12-09T01:41:28,920 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/88cacb946e2548abb2bed9357eaadf71 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/88cacb946e2548abb2bed9357eaadf71 2024-12-09T01:41:28,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/88599adc073c4d508ad3786e96bd1a42, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/8bdd8ae326e74e50a4ca21e0c01bfc01, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/dc13c71485f4497c9f6c28a184dc7efe, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/fb183aab70a847cf86f7c4281d296add, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/7c94c8d74a334bdb85821b5311af04c1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/18f41ffbf74b471783f071746d683b9b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/a35e6eb36fe34285a91ead571082ee95, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/50711638324049119da233c47265f07a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/eaa6d826128147f6a06b86579d748164, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/b240cc493ea74527859194bbb08c89f9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/e6d3a714b08140fea39f9f5e659b10b8, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/24589f63001e46099e89f7fbb3a96f67, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/bb0bf59b992b4bcea88e8dfb964d195d, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/ae2c8be98d0f4536bc18e6c10a69bacb, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/713b6efd6e0a4d04ae2017115d23e426, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/6ca47ba959db4f50bca5c39de4976c7d, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/61616af18917496b96cf1f059ebdf3a9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/aec2b9f6ef1847d58020d79ef84275d7, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/e61ec1ad015e457fa483a20f280db760, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/866c3da1c3fc434292a7b7621c3c20d5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/fb376b7847244b068726dba10f721072, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/a8f97eed45cd40c594669a110e55c305, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/b75fd758a3ad4ec1bfd0b663249f5aa7, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/f54205f4d2444943abc2370596fd7304, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/2ec5d3bcdb1742d2990c7d8aa031192b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/27444b86b0bc4c85bbc6b4b63ff154f4, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/a3b531ac61424b32be1f9c73566719f5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/51bbccb1cf1448d794796da6359b6216, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/2bf6c65c227643cab788324f3144cff6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/18513099bd58488a9bcae93cc3199ee3, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/e546655b162f477f996a1c45b30ed473, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/361feef197c44827b585a2c6b6a70827, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/c512c634069e464a999a22544dd59da6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/103cb277da454f49b6b951eaa1493875, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/3acf36f60a034bc2909a6368567b2fd6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/1c65958d076940299ae7497b9e1a39e9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/7e1ef240700b413b9571ac9e7b40c7f6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/a9c635cb49fc496b9dfabf6f7a60ddfa, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/488e48e9d5df4c9f82fcc172fd4f64f1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/e3d2a73594d5498da8684e17d968c08e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/f4e37893eab94184b9e6aab16de1ce69, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/23a0fd5b7d5c4e83a6dbf895c05a0c85, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/f14ef63966e84da38f2b81050b204725] to archive 2024-12-09T01:41:28,939 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T01:41:28,941 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/88599adc073c4d508ad3786e96bd1a42 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/88599adc073c4d508ad3786e96bd1a42 2024-12-09T01:41:28,943 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/8bdd8ae326e74e50a4ca21e0c01bfc01 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/8bdd8ae326e74e50a4ca21e0c01bfc01 2024-12-09T01:41:28,944 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/dc13c71485f4497c9f6c28a184dc7efe to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/dc13c71485f4497c9f6c28a184dc7efe 2024-12-09T01:41:28,945 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/fb183aab70a847cf86f7c4281d296add to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/fb183aab70a847cf86f7c4281d296add 2024-12-09T01:41:28,947 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/7c94c8d74a334bdb85821b5311af04c1 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/7c94c8d74a334bdb85821b5311af04c1 2024-12-09T01:41:28,948 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/18f41ffbf74b471783f071746d683b9b to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/18f41ffbf74b471783f071746d683b9b 2024-12-09T01:41:28,949 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/a35e6eb36fe34285a91ead571082ee95 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/a35e6eb36fe34285a91ead571082ee95 2024-12-09T01:41:28,952 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/50711638324049119da233c47265f07a to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/50711638324049119da233c47265f07a 2024-12-09T01:41:28,956 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/eaa6d826128147f6a06b86579d748164 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/eaa6d826128147f6a06b86579d748164 2024-12-09T01:41:28,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/b240cc493ea74527859194bbb08c89f9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/b240cc493ea74527859194bbb08c89f9 2024-12-09T01:41:28,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/e6d3a714b08140fea39f9f5e659b10b8 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/e6d3a714b08140fea39f9f5e659b10b8 2024-12-09T01:41:28,960 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/24589f63001e46099e89f7fbb3a96f67 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/24589f63001e46099e89f7fbb3a96f67 2024-12-09T01:41:28,961 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/bb0bf59b992b4bcea88e8dfb964d195d to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/bb0bf59b992b4bcea88e8dfb964d195d 2024-12-09T01:41:28,963 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/ae2c8be98d0f4536bc18e6c10a69bacb to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/ae2c8be98d0f4536bc18e6c10a69bacb 2024-12-09T01:41:28,964 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/713b6efd6e0a4d04ae2017115d23e426 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/713b6efd6e0a4d04ae2017115d23e426 2024-12-09T01:41:28,965 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/6ca47ba959db4f50bca5c39de4976c7d to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/6ca47ba959db4f50bca5c39de4976c7d 2024-12-09T01:41:28,967 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/61616af18917496b96cf1f059ebdf3a9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/61616af18917496b96cf1f059ebdf3a9 2024-12-09T01:41:28,968 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/aec2b9f6ef1847d58020d79ef84275d7 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/aec2b9f6ef1847d58020d79ef84275d7 2024-12-09T01:41:28,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/e61ec1ad015e457fa483a20f280db760 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/e61ec1ad015e457fa483a20f280db760 2024-12-09T01:41:28,971 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/866c3da1c3fc434292a7b7621c3c20d5 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/866c3da1c3fc434292a7b7621c3c20d5 2024-12-09T01:41:28,973 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/fb376b7847244b068726dba10f721072 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/fb376b7847244b068726dba10f721072 2024-12-09T01:41:28,974 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/a8f97eed45cd40c594669a110e55c305 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/a8f97eed45cd40c594669a110e55c305 2024-12-09T01:41:28,975 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/b75fd758a3ad4ec1bfd0b663249f5aa7 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/b75fd758a3ad4ec1bfd0b663249f5aa7 2024-12-09T01:41:28,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/f54205f4d2444943abc2370596fd7304 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/f54205f4d2444943abc2370596fd7304 2024-12-09T01:41:28,979 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/2ec5d3bcdb1742d2990c7d8aa031192b to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/2ec5d3bcdb1742d2990c7d8aa031192b 2024-12-09T01:41:28,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/27444b86b0bc4c85bbc6b4b63ff154f4 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/27444b86b0bc4c85bbc6b4b63ff154f4 2024-12-09T01:41:28,982 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/a3b531ac61424b32be1f9c73566719f5 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/a3b531ac61424b32be1f9c73566719f5 2024-12-09T01:41:28,983 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/51bbccb1cf1448d794796da6359b6216 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/51bbccb1cf1448d794796da6359b6216 2024-12-09T01:41:28,984 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/2bf6c65c227643cab788324f3144cff6 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/2bf6c65c227643cab788324f3144cff6 2024-12-09T01:41:28,986 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/18513099bd58488a9bcae93cc3199ee3 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/18513099bd58488a9bcae93cc3199ee3 2024-12-09T01:41:28,987 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/e546655b162f477f996a1c45b30ed473 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/e546655b162f477f996a1c45b30ed473 2024-12-09T01:41:28,989 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/361feef197c44827b585a2c6b6a70827 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/361feef197c44827b585a2c6b6a70827 2024-12-09T01:41:28,990 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/c512c634069e464a999a22544dd59da6 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/c512c634069e464a999a22544dd59da6 2024-12-09T01:41:28,995 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/103cb277da454f49b6b951eaa1493875 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/103cb277da454f49b6b951eaa1493875 2024-12-09T01:41:28,996 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/3acf36f60a034bc2909a6368567b2fd6 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/3acf36f60a034bc2909a6368567b2fd6 2024-12-09T01:41:28,998 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/1c65958d076940299ae7497b9e1a39e9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/1c65958d076940299ae7497b9e1a39e9 2024-12-09T01:41:28,999 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/7e1ef240700b413b9571ac9e7b40c7f6 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/7e1ef240700b413b9571ac9e7b40c7f6 2024-12-09T01:41:29,000 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/a9c635cb49fc496b9dfabf6f7a60ddfa to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/a9c635cb49fc496b9dfabf6f7a60ddfa 2024-12-09T01:41:29,001 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/488e48e9d5df4c9f82fcc172fd4f64f1 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/488e48e9d5df4c9f82fcc172fd4f64f1 2024-12-09T01:41:29,002 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/e3d2a73594d5498da8684e17d968c08e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/e3d2a73594d5498da8684e17d968c08e 2024-12-09T01:41:29,004 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/f4e37893eab94184b9e6aab16de1ce69 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/f4e37893eab94184b9e6aab16de1ce69 2024-12-09T01:41:29,006 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/23a0fd5b7d5c4e83a6dbf895c05a0c85 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/23a0fd5b7d5c4e83a6dbf895c05a0c85 2024-12-09T01:41:29,009 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/f14ef63966e84da38f2b81050b204725 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/f14ef63966e84da38f2b81050b204725 2024-12-09T01:41:29,011 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/0d7548d0de7b45aa9446f8af2525f358, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/afcd5d9c94424b11bf71a32aac172351, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/5cb06c2dcb524d1995af449df58877bd, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/f1415d388db04dd781f192525e5abac5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/48a6ef2782574928845a2761d501354e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/763fbaa0f3f84e23b621ffa2c7971c74, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/22567e1ff4c743e9beb281fd3fdb4eb1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/12825a21d8624e6899044aa5bf1cafb2, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/62f6339c290b45afb2243d432a889f4a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/0f4c51d39dd344d0936abb6e8475535a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/91332d98587d45b28851d11b5bd3ea16, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/c7ef9518edd04457a1bb158b9cc4ad0e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/055f71331c2842aba3f44576f91439a5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/5f55ffbac1fb49149840b4380b5506b6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/d665d3f5011b48fbacd94db86d7d50d6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/de56646fa92242cf90614c3d77bc5d89, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/3b574b2b58e84ae494e04067ece9c1f1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/ad1c1fbc4d2f4d339bdd7185352d07eb, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/2165e34413384c51a4d3eb5b3d9cf370, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/540b5ec51f604e87baefca90fe327236, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/c4da6552ec6d45979faceedb34848577, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/1e406b66a3da40c4860d7db43c8090f2, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/1231e3887709433cb1420064b692ab3c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/24869ba34af643009caed653b16d6b6e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/d2b0656bafee4174afa94ca6d6c85d21, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/643e5d5c827a49968e85bf81190993ac, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/2c78734d4116417fb42c53ba6482ee1a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/3c60ff22a79b4ea6b7312bc93efbcda6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/8567cec77b0f4a11a315f2709431865f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/87cee428298449e5b21a0c388236256d, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/77f235daa36645c692eede49532f5364, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/1090b4a26c2447479a42c4c122e03470, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/bc5f53f6935a4f918f7c05b8ee7d0e78, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/c5c172e5984f46dfa8b070b2487f93a2, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/23ee3674e0194a138911397a9ec6f257, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/a9b553ddc1f24726b6d3eb54506e8bdf, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/79a7cb563d2b41eb95fa0054b7d49fe3, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/5364d1740f994ea28ea0e7921168ded0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/941d5d375bed44d78403b123cc8832a0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/ec1060b1c4c849298599f46b5eebc5cd, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/1ce3cbb25fcc4885bdc1264b7a19f88b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/536444b497e34db4ab5860425b00a3f0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/57e8fce0bb73413a9be8718bdcc08fac] to archive 2024-12-09T01:41:29,013 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T01:41:29,015 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/0d7548d0de7b45aa9446f8af2525f358 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/0d7548d0de7b45aa9446f8af2525f358 2024-12-09T01:41:29,016 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/afcd5d9c94424b11bf71a32aac172351 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/afcd5d9c94424b11bf71a32aac172351 2024-12-09T01:41:29,018 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/5cb06c2dcb524d1995af449df58877bd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/5cb06c2dcb524d1995af449df58877bd 2024-12-09T01:41:29,019 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/f1415d388db04dd781f192525e5abac5 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/f1415d388db04dd781f192525e5abac5 2024-12-09T01:41:29,021 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/48a6ef2782574928845a2761d501354e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/48a6ef2782574928845a2761d501354e 2024-12-09T01:41:29,026 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/763fbaa0f3f84e23b621ffa2c7971c74 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/763fbaa0f3f84e23b621ffa2c7971c74 2024-12-09T01:41:29,028 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/22567e1ff4c743e9beb281fd3fdb4eb1 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/22567e1ff4c743e9beb281fd3fdb4eb1 2024-12-09T01:41:29,029 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/12825a21d8624e6899044aa5bf1cafb2 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/12825a21d8624e6899044aa5bf1cafb2 2024-12-09T01:41:29,031 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/62f6339c290b45afb2243d432a889f4a to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/62f6339c290b45afb2243d432a889f4a 2024-12-09T01:41:29,033 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/0f4c51d39dd344d0936abb6e8475535a to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/0f4c51d39dd344d0936abb6e8475535a 2024-12-09T01:41:29,034 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/91332d98587d45b28851d11b5bd3ea16 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/91332d98587d45b28851d11b5bd3ea16 2024-12-09T01:41:29,036 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/c7ef9518edd04457a1bb158b9cc4ad0e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/c7ef9518edd04457a1bb158b9cc4ad0e 2024-12-09T01:41:29,037 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/055f71331c2842aba3f44576f91439a5 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/055f71331c2842aba3f44576f91439a5 2024-12-09T01:41:29,038 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/5f55ffbac1fb49149840b4380b5506b6 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/5f55ffbac1fb49149840b4380b5506b6 2024-12-09T01:41:29,040 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/d665d3f5011b48fbacd94db86d7d50d6 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/d665d3f5011b48fbacd94db86d7d50d6 2024-12-09T01:41:29,041 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/de56646fa92242cf90614c3d77bc5d89 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/de56646fa92242cf90614c3d77bc5d89 2024-12-09T01:41:29,043 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/3b574b2b58e84ae494e04067ece9c1f1 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/3b574b2b58e84ae494e04067ece9c1f1 2024-12-09T01:41:29,045 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/ad1c1fbc4d2f4d339bdd7185352d07eb to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/ad1c1fbc4d2f4d339bdd7185352d07eb 2024-12-09T01:41:29,046 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/2165e34413384c51a4d3eb5b3d9cf370 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/2165e34413384c51a4d3eb5b3d9cf370 2024-12-09T01:41:29,047 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/540b5ec51f604e87baefca90fe327236 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/540b5ec51f604e87baefca90fe327236 2024-12-09T01:41:29,048 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/c4da6552ec6d45979faceedb34848577 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/c4da6552ec6d45979faceedb34848577 2024-12-09T01:41:29,049 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/1e406b66a3da40c4860d7db43c8090f2 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/1e406b66a3da40c4860d7db43c8090f2 2024-12-09T01:41:29,051 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/1231e3887709433cb1420064b692ab3c to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/1231e3887709433cb1420064b692ab3c 2024-12-09T01:41:29,052 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/24869ba34af643009caed653b16d6b6e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/24869ba34af643009caed653b16d6b6e 2024-12-09T01:41:29,053 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/d2b0656bafee4174afa94ca6d6c85d21 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/d2b0656bafee4174afa94ca6d6c85d21 2024-12-09T01:41:29,055 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/643e5d5c827a49968e85bf81190993ac to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/643e5d5c827a49968e85bf81190993ac 2024-12-09T01:41:29,056 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/2c78734d4116417fb42c53ba6482ee1a to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/2c78734d4116417fb42c53ba6482ee1a 2024-12-09T01:41:29,058 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/3c60ff22a79b4ea6b7312bc93efbcda6 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/3c60ff22a79b4ea6b7312bc93efbcda6 2024-12-09T01:41:29,059 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/8567cec77b0f4a11a315f2709431865f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/8567cec77b0f4a11a315f2709431865f 2024-12-09T01:41:29,061 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/87cee428298449e5b21a0c388236256d to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/87cee428298449e5b21a0c388236256d 2024-12-09T01:41:29,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/77f235daa36645c692eede49532f5364 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/77f235daa36645c692eede49532f5364 2024-12-09T01:41:29,065 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/1090b4a26c2447479a42c4c122e03470 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/1090b4a26c2447479a42c4c122e03470 2024-12-09T01:41:29,068 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/bc5f53f6935a4f918f7c05b8ee7d0e78 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/bc5f53f6935a4f918f7c05b8ee7d0e78 2024-12-09T01:41:29,071 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/c5c172e5984f46dfa8b070b2487f93a2 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/c5c172e5984f46dfa8b070b2487f93a2 2024-12-09T01:41:29,073 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/23ee3674e0194a138911397a9ec6f257 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/23ee3674e0194a138911397a9ec6f257 2024-12-09T01:41:29,074 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/a9b553ddc1f24726b6d3eb54506e8bdf to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/a9b553ddc1f24726b6d3eb54506e8bdf 2024-12-09T01:41:29,075 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/79a7cb563d2b41eb95fa0054b7d49fe3 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/79a7cb563d2b41eb95fa0054b7d49fe3 2024-12-09T01:41:29,076 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/5364d1740f994ea28ea0e7921168ded0 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/5364d1740f994ea28ea0e7921168ded0 2024-12-09T01:41:29,077 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/941d5d375bed44d78403b123cc8832a0 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/941d5d375bed44d78403b123cc8832a0 2024-12-09T01:41:29,079 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/ec1060b1c4c849298599f46b5eebc5cd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/ec1060b1c4c849298599f46b5eebc5cd 2024-12-09T01:41:29,080 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/1ce3cbb25fcc4885bdc1264b7a19f88b to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/1ce3cbb25fcc4885bdc1264b7a19f88b 2024-12-09T01:41:29,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/536444b497e34db4ab5860425b00a3f0 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/536444b497e34db4ab5860425b00a3f0 2024-12-09T01:41:29,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/57e8fce0bb73413a9be8718bdcc08fac to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/57e8fce0bb73413a9be8718bdcc08fac 2024-12-09T01:41:29,101 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/recovered.edits/697.seqid, newMaxSeqId=697, maxSeqId=1 2024-12-09T01:41:29,102 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. 2024-12-09T01:41:29,102 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 88980a4de80c5830889346818652c3a7: Waiting for close lock at 1733708487521Running coprocessor pre-close hooks at 1733708487521Disabling compacts and flushes for region at 1733708487521Disabling writes for close at 1733708487521Obtaining lock to block concurrent updates at 1733708487521Preparing flush snapshotting stores in 88980a4de80c5830889346818652c3a7 at 1733708487521Finished memstore snapshotting TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733708487522 (+1 ms)Flushing stores of TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7. at 1733708487523 (+1 ms)Flushing 88980a4de80c5830889346818652c3a7/A: creating writer at 1733708487523Flushing 88980a4de80c5830889346818652c3a7/A: appending metadata at 1733708487529 (+6 ms)Flushing 88980a4de80c5830889346818652c3a7/A: closing flushed file at 1733708487529Flushing 88980a4de80c5830889346818652c3a7/B: creating writer at 1733708487941 (+412 ms)Flushing 88980a4de80c5830889346818652c3a7/B: appending metadata at 1733708487945 (+4 ms)Flushing 88980a4de80c5830889346818652c3a7/B: closing flushed file at 1733708487945Flushing 88980a4de80c5830889346818652c3a7/C: creating writer at 1733708488358 (+413 ms)Flushing 88980a4de80c5830889346818652c3a7/C: appending metadata at 1733708488362 (+4 ms)Flushing 88980a4de80c5830889346818652c3a7/C: closing flushed file at 1733708488362Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@475454ac: reopening flushed file at 1733708488793 (+431 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@196554ed: reopening flushed file at 1733708488799 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@195aae9d: reopening flushed file at 1733708488807 (+8 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 88980a4de80c5830889346818652c3a7 in 1294ms, sequenceid=694, compaction requested=false at 1733708488815 (+8 ms)Writing region close event to WAL at 1733708489096 (+281 ms)Running coprocessor post-close hooks at 1733708489101 (+5 ms)Closed at 1733708489101 2024-12-09T01:41:29,104 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 88980a4de80c5830889346818652c3a7 2024-12-09T01:41:29,105 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=88980a4de80c5830889346818652c3a7, regionState=CLOSED 2024-12-09T01:41:29,107 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 because future has completed 2024-12-09T01:41:29,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-12-09T01:41:29,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 88980a4de80c5830889346818652c3a7, server=ef6f18c58dc9,38627,1733708453943 in 1.7450 sec 2024-12-09T01:41:29,113 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-12-09T01:41:29,113 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88980a4de80c5830889346818652c3a7, UNASSIGN in 1.7530 sec 2024-12-09T01:41:29,116 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-09T01:41:29,116 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7610 sec 2024-12-09T01:41:29,118 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708489117"}]},"ts":"1733708489117"} 2024-12-09T01:41:29,119 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T01:41:29,119 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T01:41:29,121 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.7770 sec 2024-12-09T01:41:29,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-09T01:41:29,494 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:41:29,499 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T01:41:29,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:41:29,504 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:41:29,506 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:41:29,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-09T01:41:29,514 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7 2024-12-09T01:41:29,519 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A, FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B, FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C, FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/recovered.edits] 2024-12-09T01:41:29,522 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/63c46457ffbd473685a0db855c0c9caa to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/63c46457ffbd473685a0db855c0c9caa 2024-12-09T01:41:29,523 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ae21897c0cbd48cd963f228f3a9004d7 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/A/ae21897c0cbd48cd963f228f3a9004d7 2024-12-09T01:41:29,527 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/6b27999909024f14b29bd880b3725c7f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/6b27999909024f14b29bd880b3725c7f 2024-12-09T01:41:29,528 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/c7ecc089ae72441e82ed2a514de59fdf to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/B/c7ecc089ae72441e82ed2a514de59fdf 2024-12-09T01:41:29,531 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/98157620e15b4fd0be19440a0b3f16f6 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/98157620e15b4fd0be19440a0b3f16f6 2024-12-09T01:41:29,532 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/c524ca0a24564060af561e35719f2b23 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/C/c524ca0a24564060af561e35719f2b23 2024-12-09T01:41:29,535 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/recovered.edits/697.seqid to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7/recovered.edits/697.seqid 2024-12-09T01:41:29,536 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/88980a4de80c5830889346818652c3a7 2024-12-09T01:41:29,536 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T01:41:29,543 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:41:29,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-09T01:41:29,553 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T01:41:29,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-09T01:41:29,620 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T01:41:29,622 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:41:29,622 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T01:41:29,623 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733708489622"}]},"ts":"9223372036854775807"} 2024-12-09T01:41:29,627 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T01:41:29,627 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 88980a4de80c5830889346818652c3a7, NAME => 'TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T01:41:29,628 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T01:41:29,628 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733708489628"}]},"ts":"9223372036854775807"} 2024-12-09T01:41:29,632 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T01:41:29,633 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:41:29,635 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 134 msec 2024-12-09T01:41:29,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-12-09T01:41:29,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T01:41:29,824 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:41:29,836 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=247 (was 211) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;ef6f18c58dc9:38627-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1866703801_22 at /127.0.0.1:52658 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=487 (was 446) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=12469 (was 12528) 2024-12-09T01:41:29,848 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=247, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=487, ProcessCount=11, AvailableMemoryMB=12468 2024-12-09T01:41:29,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T01:41:29,850 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:41:29,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T01:41:29,853 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T01:41:29,853 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:29,853 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-12-09T01:41:29,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-09T01:41:29,854 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T01:41:29,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741972_1148 (size=1061) 2024-12-09T01:41:29,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-09T01:41:30,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-09T01:41:30,265 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb 2024-12-09T01:41:30,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741973_1149 (size=53) 2024-12-09T01:41:30,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-09T01:41:30,675 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:41:30,675 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ca70d838c08a81d31de52aceb8d05bfd, disabling compactions & flushes 2024-12-09T01:41:30,675 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:30,675 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:30,675 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. after waiting 0 ms 2024-12-09T01:41:30,675 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:30,675 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:30,675 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ca70d838c08a81d31de52aceb8d05bfd: Waiting for close lock at 1733708490675Disabling compacts and flushes for region at 1733708490675Disabling writes for close at 1733708490675Writing region close event to WAL at 1733708490675Closed at 1733708490675 2024-12-09T01:41:30,677 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T01:41:30,677 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733708490677"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733708490677"}]},"ts":"1733708490677"} 2024-12-09T01:41:30,679 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T01:41:30,680 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T01:41:30,681 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708490680"}]},"ts":"1733708490680"} 2024-12-09T01:41:30,682 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T01:41:30,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca70d838c08a81d31de52aceb8d05bfd, ASSIGN}] 2024-12-09T01:41:30,684 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca70d838c08a81d31de52aceb8d05bfd, ASSIGN 2024-12-09T01:41:30,684 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca70d838c08a81d31de52aceb8d05bfd, ASSIGN; state=OFFLINE, location=ef6f18c58dc9,38627,1733708453943; forceNewPlan=false, retain=false 2024-12-09T01:41:30,835 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=ca70d838c08a81d31de52aceb8d05bfd, regionState=OPENING, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:30,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca70d838c08a81d31de52aceb8d05bfd, ASSIGN because future has completed 2024-12-09T01:41:30,838 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943}] 2024-12-09T01:41:30,993 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:30,993 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:41:30,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-09T01:41:30,994 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:30,994 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:41:30,994 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:30,994 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:30,996 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:30,997 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:41:30,998 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca70d838c08a81d31de52aceb8d05bfd columnFamilyName A 2024-12-09T01:41:30,999 DEBUG [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:30,999 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] regionserver.HStore(327): Store=ca70d838c08a81d31de52aceb8d05bfd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:41:30,999 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:31,001 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:41:31,001 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca70d838c08a81d31de52aceb8d05bfd columnFamilyName B 2024-12-09T01:41:31,001 DEBUG [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:31,001 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] regionserver.HStore(327): Store=ca70d838c08a81d31de52aceb8d05bfd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:41:31,001 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:31,002 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:41:31,003 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca70d838c08a81d31de52aceb8d05bfd columnFamilyName C 2024-12-09T01:41:31,003 DEBUG [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:31,003 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] regionserver.HStore(327): Store=ca70d838c08a81d31de52aceb8d05bfd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:41:31,004 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:31,004 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:31,004 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:31,005 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:31,006 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:31,006 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:31,007 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T01:41:31,008 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:31,010 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:41:31,011 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened ca70d838c08a81d31de52aceb8d05bfd; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61990455, jitterRate=-0.07627023756504059}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T01:41:31,011 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:31,012 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for ca70d838c08a81d31de52aceb8d05bfd: Running coprocessor pre-open hook at 1733708490995Writing region info on filesystem at 1733708490995Initializing all the Stores at 1733708490995Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708490996 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708490996Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708490996Cleaning up temporary data from old regions at 1733708491006 (+10 ms)Running coprocessor post-open hooks at 1733708491011 (+5 ms)Region opened successfully at 1733708491011 2024-12-09T01:41:31,012 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., pid=36, masterSystemTime=1733708490989 2024-12-09T01:41:31,015 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:31,015 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:31,015 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=ca70d838c08a81d31de52aceb8d05bfd, regionState=OPEN, openSeqNum=2, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:31,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 because future has completed 2024-12-09T01:41:31,020 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-09T01:41:31,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 in 182 msec 2024-12-09T01:41:31,022 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-12-09T01:41:31,022 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca70d838c08a81d31de52aceb8d05bfd, ASSIGN in 339 msec 2024-12-09T01:41:31,023 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T01:41:31,024 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708491023"}]},"ts":"1733708491023"} 2024-12-09T01:41:31,025 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T01:41:31,026 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T01:41:31,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1760 sec 2024-12-09T01:41:32,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-09T01:41:32,005 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:41:32,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149f2ae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,006 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:41:32,006 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:41:32,008 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:41:32,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:41:32,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:41:32,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@281ee23d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:41:32,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:41:32,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:32,011 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:41:32,012 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58e15a65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,012 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:41:32,013 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:41:32,014 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:41:32,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60514, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:41:32,017 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T01:41:32,018 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is ef6f18c58dc9,37995,1733708453073 2024-12-09T01:41:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3610e814 2024-12-09T01:41:32,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T01:41:32,020 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T01:41:32,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T01:41:32,027 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:41:32,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-09T01:41:32,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741974_1150 (size=1097) 2024-12-09T01:41:32,050 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-09T01:41:32,050 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-09T01:41:32,055 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T01:41:32,064 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca70d838c08a81d31de52aceb8d05bfd, REOPEN/MOVE}] 2024-12-09T01:41:32,065 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca70d838c08a81d31de52aceb8d05bfd, REOPEN/MOVE 2024-12-09T01:41:32,066 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=ca70d838c08a81d31de52aceb8d05bfd, regionState=CLOSING, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:32,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca70d838c08a81d31de52aceb8d05bfd, REOPEN/MOVE because future has completed 2024-12-09T01:41:32,076 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T01:41:32,077 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943}] 2024-12-09T01:41:32,230 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:32,230 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T01:41:32,231 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing ca70d838c08a81d31de52aceb8d05bfd, disabling compactions & flushes 2024-12-09T01:41:32,231 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:32,231 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:32,231 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. after waiting 0 ms 2024-12-09T01:41:32,231 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:32,244 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T01:41:32,245 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:32,246 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for ca70d838c08a81d31de52aceb8d05bfd: Waiting for close lock at 1733708492230Running coprocessor pre-close hooks at 1733708492230Disabling compacts and flushes for region at 1733708492230Disabling writes for close at 1733708492231 (+1 ms)Writing region close event to WAL at 1733708492231Running coprocessor post-close hooks at 1733708492245 (+14 ms)Closed at 1733708492245 2024-12-09T01:41:32,246 WARN [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: ca70d838c08a81d31de52aceb8d05bfd to self. 2024-12-09T01:41:32,251 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=ca70d838c08a81d31de52aceb8d05bfd, regionState=CLOSED 2024-12-09T01:41:32,252 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:32,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 because future has completed 2024-12-09T01:41:32,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-12-09T01:41:32,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 in 178 msec 2024-12-09T01:41:32,258 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca70d838c08a81d31de52aceb8d05bfd, REOPEN/MOVE; state=CLOSED, location=ef6f18c58dc9,38627,1733708453943; forceNewPlan=false, retain=true 2024-12-09T01:41:32,409 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=ca70d838c08a81d31de52aceb8d05bfd, regionState=OPENING, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:32,412 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca70d838c08a81d31de52aceb8d05bfd, REOPEN/MOVE because future has completed 2024-12-09T01:41:32,413 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943}] 2024-12-09T01:41:32,569 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:32,570 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:41:32,570 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:32,570 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:41:32,570 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:32,570 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:32,573 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:32,573 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:41:32,578 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca70d838c08a81d31de52aceb8d05bfd columnFamilyName A 2024-12-09T01:41:32,580 DEBUG [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:32,580 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] regionserver.HStore(327): Store=ca70d838c08a81d31de52aceb8d05bfd/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:41:32,581 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:32,581 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:41:32,581 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca70d838c08a81d31de52aceb8d05bfd columnFamilyName B 2024-12-09T01:41:32,581 DEBUG [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:32,582 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] regionserver.HStore(327): Store=ca70d838c08a81d31de52aceb8d05bfd/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:41:32,582 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:32,582 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:41:32,583 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca70d838c08a81d31de52aceb8d05bfd columnFamilyName C 2024-12-09T01:41:32,583 DEBUG [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:32,583 INFO [StoreOpener-ca70d838c08a81d31de52aceb8d05bfd-1 {}] regionserver.HStore(327): Store=ca70d838c08a81d31de52aceb8d05bfd/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:41:32,583 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:32,583 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:32,584 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:32,585 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:32,586 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:32,586 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:32,586 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T01:41:32,588 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:32,588 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened ca70d838c08a81d31de52aceb8d05bfd; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65881507, jitterRate=-0.018289044499397278}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T01:41:32,588 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:32,590 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for ca70d838c08a81d31de52aceb8d05bfd: Running coprocessor pre-open hook at 1733708492570Writing region info on filesystem at 1733708492570Initializing all the Stores at 1733708492571 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708492571Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708492571Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708492571Cleaning up temporary data from old regions at 1733708492586 (+15 ms)Running coprocessor post-open hooks at 1733708492588 (+2 ms)Region opened successfully at 1733708492590 (+2 ms) 2024-12-09T01:41:32,590 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., pid=41, masterSystemTime=1733708492566 2024-12-09T01:41:32,592 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:32,592 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:32,593 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=ca70d838c08a81d31de52aceb8d05bfd, regionState=OPEN, openSeqNum=5, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:32,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 because future has completed 2024-12-09T01:41:32,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-12-09T01:41:32,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 in 182 msec 2024-12-09T01:41:32,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-12-09T01:41:32,599 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca70d838c08a81d31de52aceb8d05bfd, REOPEN/MOVE in 533 msec 2024-12-09T01:41:32,601 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-12-09T01:41:32,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 546 msec 2024-12-09T01:41:32,605 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 572 msec 2024-12-09T01:41:32,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-12-09T01:41:32,607 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:41:32,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62e0a2fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,608 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:41:32,608 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:41:32,609 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:41:32,609 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:41:32,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:41:32,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e66cd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:41:32,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:41:32,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:32,611 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:41:32,612 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55d9b2e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,613 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4716031f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,613 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:41:32,613 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:41:32,614 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:41:32,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:41:32,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:41:32,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ffdc4e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:41:32,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:41:32,614 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:32,615 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43834, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:41:32,616 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bace19e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15d3ba43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,617 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:41:32,617 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:41:32,617 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:41:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:41:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:41:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bd267d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:41:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:41:32,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:32,619 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:41:32,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41fb87bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aa73915, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,621 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:41:32,621 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:41:32,621 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:41:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:41:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:41:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d800124, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:41:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:41:32,622 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:32,623 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:41:32,623 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@345815a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47250f91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,624 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:41:32,624 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:41:32,625 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:41:32,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:41:32,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:41:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16390f18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:41:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:41:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:32,627 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:41:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d1b6a70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e890ec6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,628 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:41:32,628 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:41:32,629 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:41:32,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:41:32,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:41:32,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d774576, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:41:32,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:41:32,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:32,631 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:41:32,631 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@491e1800, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ed913a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,632 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:41:32,632 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:41:32,633 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:41:32,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:41:32,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:41:32,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dbf4eab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:41:32,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:41:32,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:32,634 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:41:32,635 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4225cb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49519de2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,636 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:41:32,636 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:41:32,636 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:41:32,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:41:32,636 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:41:32,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f981f5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:41:32,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:41:32,637 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:32,638 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:41:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55cd1319, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16801009, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,639 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:41:32,639 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:41:32,639 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:41:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:41:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:41:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46e137ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:41:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:41:32,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:32,641 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:41:32,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3649d1c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:41:32,642 DEBUG [Thread-746 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:41:32,643 DEBUG [Thread-737 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:41:32,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:41:32,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:41:32,644 DEBUG [Thread-750 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:41:32,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:41:32,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:41:32,644 DEBUG [Thread-741 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:41:32,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:41:32,645 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:41:32,645 DEBUG [Thread-739 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:41:32,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:41:32,646 DEBUG [Thread-744 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:41:32,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:41:32,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:41:32,646 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:41:32,647 DEBUG [Thread-733 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:41:32,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:41:32,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:41:32,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:41:32,647 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:41:32,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:41:32,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:41:32,648 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:41:32,649 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:41:32,649 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:41:32,649 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:41:32,650 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:41:32,652 DEBUG [Thread-748 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:41:32,653 DEBUG [Thread-735 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:41:32,653 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:41:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-12-09T01:41:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:41:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:41:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:41:32,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:41:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T01:41:32,660 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:41:32,660 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:41:32,660 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:41:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:41:32,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:41:32,662 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:41:32,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:41:32,663 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:41:32,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:41:32,664 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:41:32,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:41:32,665 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:41:32,665 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:41:32,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:41:32,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:32,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca70d838c08a81d31de52aceb8d05bfd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:41:32,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=A 2024-12-09T01:41:32,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:32,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=B 2024-12-09T01:41:32,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:32,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=C 2024-12-09T01:41:32,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:32,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120971b8cd797c3246ee95fbc9154328e8d2_ca70d838c08a81d31de52aceb8d05bfd is 50, key is test_row_0/A:col10/1733708492669/Put/seqid=0 2024-12-09T01:41:32,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741975_1151 (size=12154) 2024-12-09T01:41:32,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708492732, max timestamp=1733708492752Num uniques 859; 2024-12-09T01:41:32,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708492732, max timestamp=1733708492752Num uniques 859; 2024-12-09T01:41:32,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708492732, max timestamp=1733708492752Num uniques 859; 2024-12-09T01:41:32,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:41:32,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:41:32,758 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:41:32,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T01:41:32,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708552790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:32,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:32,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:32,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:32,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708552791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:32,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:32,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708552802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:32,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:32,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:32,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:32,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T01:41:32,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:32,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:32,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:32,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:32,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:32,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:32,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:32,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708552809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:32,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:32,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:32,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:32,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708552818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:32,823 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:32,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:32,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:32,824 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:32,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:32,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708552913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:32,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:32,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:32,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:32,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:32,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708552924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:32,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:32,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:32,932 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:32,932 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:32,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708552924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:32,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:32,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:32,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:32,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:32,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708552933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:32,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:32,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:32,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:32,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:32,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708552934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:32,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:32,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T01:41:32,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T01:41:32,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:32,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:32,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:32,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:32,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:32,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:33,129 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T01:41:33,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:33,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:33,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:33,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:33,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:33,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:33,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:33,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708553133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:33,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:33,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:33,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708553134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:33,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:33,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:33,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708553144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:33,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:33,149 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:33,155 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120971b8cd797c3246ee95fbc9154328e8d2_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120971b8cd797c3246ee95fbc9154328e8d2_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:33,157 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/ba7b6217a631419cbb8bba6eff13a391, store: [table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:33,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:33,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708553154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:33,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,163 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:33,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:33,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708553154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:33,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:33,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/ba7b6217a631419cbb8bba6eff13a391 is 175, key is test_row_0/A:col10/1733708492669/Put/seqid=0 2024-12-09T01:41:33,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741976_1152 (size=30955) 2024-12-09T01:41:33,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T01:41:33,284 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T01:41:33,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:33,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:33,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:33,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:33,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:33,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:33,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T01:41:33,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:33,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:33,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:33,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:33,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:33,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:33,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:33,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708553444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:33,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,449 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:33,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:33,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708553444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:33,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:33,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:33,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708553453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:33,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:33,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708553474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:33,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:33,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:33,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708553474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:33,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:33,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T01:41:33,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:33,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:33,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:33,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:33,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:33,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:33,635 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/ba7b6217a631419cbb8bba6eff13a391 2024-12-09T01:41:33,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/5f25f9dd97e6408ca6ba863add174329 is 50, key is test_row_0/B:col10/1733708492669/Put/seqid=0 2024-12-09T01:41:33,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741977_1153 (size=12001) 2024-12-09T01:41:33,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T01:41:33,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:33,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:33,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:33,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:33,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:33,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:33,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T01:41:33,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T01:41:33,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:33,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:33,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:33,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:33,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:33,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708553953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:33,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:33,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:33,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708553953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:33,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:33,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:33,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708553974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:33,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:33,986 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:33,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708553983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:33,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:33,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708553983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:33,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:33,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:33,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:34,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T01:41:34,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:34,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:34,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:34,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:34,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:34,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:34,114 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/5f25f9dd97e6408ca6ba863add174329 2024-12-09T01:41:34,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/6f37d1eb87dc4f01ba589628f3007560 is 50, key is test_row_0/C:col10/1733708492669/Put/seqid=0 2024-12-09T01:41:34,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T01:41:34,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:34,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:34,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:34,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:34,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:34,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:34,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741978_1154 (size=12001) 2024-12-09T01:41:34,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/6f37d1eb87dc4f01ba589628f3007560 2024-12-09T01:41:34,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/ba7b6217a631419cbb8bba6eff13a391 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/ba7b6217a631419cbb8bba6eff13a391 2024-12-09T01:41:34,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/ba7b6217a631419cbb8bba6eff13a391, entries=150, sequenceid=15, filesize=30.2 K 2024-12-09T01:41:34,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/5f25f9dd97e6408ca6ba863add174329 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/5f25f9dd97e6408ca6ba863add174329 2024-12-09T01:41:34,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/5f25f9dd97e6408ca6ba863add174329, entries=150, sequenceid=15, filesize=11.7 K 2024-12-09T01:41:34,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/6f37d1eb87dc4f01ba589628f3007560 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/6f37d1eb87dc4f01ba589628f3007560 2024-12-09T01:41:34,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/6f37d1eb87dc4f01ba589628f3007560, entries=150, sequenceid=15, filesize=11.7 K 2024-12-09T01:41:34,270 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for ca70d838c08a81d31de52aceb8d05bfd in 1587ms, sequenceid=15, compaction requested=false 2024-12-09T01:41:34,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:34,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-12-09T01:41:34,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:34,368 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing ca70d838c08a81d31de52aceb8d05bfd 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-09T01:41:34,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=A 2024-12-09T01:41:34,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:34,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=B 2024-12-09T01:41:34,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:34,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=C 2024-12-09T01:41:34,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:34,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120913980a38463a407d92020113c17d31b2_ca70d838c08a81d31de52aceb8d05bfd is 50, key is test_row_0/A:col10/1733708492804/Put/seqid=0 2024-12-09T01:41:34,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741979_1155 (size=12154) 2024-12-09T01:41:34,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,415 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120913980a38463a407d92020113c17d31b2_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120913980a38463a407d92020113c17d31b2_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:34,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/5dd60be42f644445a0e6b7e4e048121a, store: [table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:34,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/5dd60be42f644445a0e6b7e4e048121a is 175, key is test_row_0/A:col10/1733708492804/Put/seqid=0 2024-12-09T01:41:34,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741980_1156 (size=30955) 2024-12-09T01:41:34,429 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/5dd60be42f644445a0e6b7e4e048121a 2024-12-09T01:41:34,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/9ef94889f8fe418da1d7390e0898ff51 is 50, key is test_row_0/B:col10/1733708492804/Put/seqid=0 2024-12-09T01:41:34,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741981_1157 (size=12001) 2024-12-09T01:41:34,482 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/9ef94889f8fe418da1d7390e0898ff51 2024-12-09T01:41:34,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/a88fb51330f244b795bbf1832d831013 is 50, key is test_row_0/C:col10/1733708492804/Put/seqid=0 2024-12-09T01:41:34,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741982_1158 (size=12001) 2024-12-09T01:41:34,510 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/a88fb51330f244b795bbf1832d831013 2024-12-09T01:41:34,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/5dd60be42f644445a0e6b7e4e048121a as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5dd60be42f644445a0e6b7e4e048121a 2024-12-09T01:41:34,540 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5dd60be42f644445a0e6b7e4e048121a, entries=150, sequenceid=52, filesize=30.2 K 2024-12-09T01:41:34,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/9ef94889f8fe418da1d7390e0898ff51 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/9ef94889f8fe418da1d7390e0898ff51 2024-12-09T01:41:34,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,551 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/9ef94889f8fe418da1d7390e0898ff51, entries=150, sequenceid=52, filesize=11.7 K 2024-12-09T01:41:34,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/a88fb51330f244b795bbf1832d831013 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/a88fb51330f244b795bbf1832d831013 2024-12-09T01:41:34,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,562 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/a88fb51330f244b795bbf1832d831013, entries=150, sequenceid=52, filesize=11.7 K 2024-12-09T01:41:34,565 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=0 B/0 for ca70d838c08a81d31de52aceb8d05bfd in 196ms, sequenceid=52, compaction requested=false 2024-12-09T01:41:34,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:34,565 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:34,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-12-09T01:41:34,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-12-09T01:41:34,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,571 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-12-09T01:41:34,571 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9030 sec 2024-12-09T01:41:34,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.9180 sec 2024-12-09T01:41:34,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,690 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T01:41:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-09T01:41:34,805 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:41:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,806 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:41:34,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-12-09T01:41:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,808 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:41:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,809 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:41:34,809 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:41:34,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T01:41:34,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T01:41:34,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-12-09T01:41:34,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:34,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:34,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:34,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-12-09T01:41:34,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-12-09T01:41:34,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-12-09T01:41:34,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-12-09T01:41:34,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,971 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 163 msec 2024-12-09T01:41:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:35,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca70d838c08a81d31de52aceb8d05bfd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:41:35,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=A 2024-12-09T01:41:35,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:35,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=B 2024-12-09T01:41:35,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:35,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=C 2024-12-09T01:41:35,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:35,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092a01d397078649a8a1a7d41cac2f2b5f_ca70d838c08a81d31de52aceb8d05bfd is 50, key is test_row_0/A:col10/1733708495085/Put/seqid=0 2024-12-09T01:41:35,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-12-09T01:41:35,124 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:41:35,125 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:41:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-12-09T01:41:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T01:41:35,128 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:41:35,129 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:41:35,129 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:41:35,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741983_1159 (size=19474) 2024-12-09T01:41:35,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708495114, max timestamp=1733708495147Num uniques 859; 2024-12-09T01:41:35,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:41:35,153 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:35,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733708495114, max timestamp=1733708495147Num uniques 859; 2024-12-09T01:41:35,155 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T01:41:35,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733708495114, max timestamp=1733708495147Num uniques 859; 2024-12-09T01:41:35,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T01:41:35,165 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092a01d397078649a8a1a7d41cac2f2b5f_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092a01d397078649a8a1a7d41cac2f2b5f_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:35,166 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/5593f999b32f48b29e35237f114a53ee, store: [table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:35,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/5593f999b32f48b29e35237f114a53ee is 175, key is test_row_0/A:col10/1733708495085/Put/seqid=0 2024-12-09T01:41:35,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741984_1160 (size=56733) 2024-12-09T01:41:35,212 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/5593f999b32f48b29e35237f114a53ee 2024-12-09T01:41:35,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:35,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708555205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:35,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733708495186, max timestamp=1733708495228Num uniques 1018; 2024-12-09T01:41:35,233 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-12-09T01:41:35,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T01:41:35,237 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733708495186, max timestamp=1733708495228Num uniques 1009; 2024-12-09T01:41:35,238 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-09T01:41:35,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/f00ec6e685d74a048bd6d1c13429d4e0 is 50, key is test_row_0/B:col10/1733708495085/Put/seqid=0 2024-12-09T01:41:35,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733708495186, max timestamp=1733708495225Num uniques 1009; 2024-12-09T01:41:35,240 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=7688, suffix heap size=132904,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=168 2024-12-09T01:41:35,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741985_1161 (size=12001) 2024-12-09T01:41:35,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T01:41:35,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:35,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:35,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:35,293 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:35,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:35,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:35,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:35,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708555289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:35,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:35,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:35,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708555290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708555292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:35,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,308 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:35,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:35,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:35,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708555306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:35,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:35,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:35,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708555335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:35,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:35,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708555413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:35,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:35,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708555414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:35,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:35,421 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708555414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:35,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,424 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:35,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708555424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:35,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T01:41:35,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T01:41:35,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:35,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:35,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:35,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:35,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:35,565 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708555553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:35,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:35,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T01:41:35,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:35,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:35,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:35,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:35,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:35,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:35,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708555625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:35,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:35,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708555634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:35,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:35,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708555634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:35,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:35,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:35,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708555644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:35,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:35,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/f00ec6e685d74a048bd6d1c13429d4e0 2024-12-09T01:41:35,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/e160020cd52148889093e85ee8fb8227 is 50, key is test_row_0/C:col10/1733708495085/Put/seqid=0 2024-12-09T01:41:35,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741986_1162 (size=12001) 2024-12-09T01:41:35,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/e160020cd52148889093e85ee8fb8227 2024-12-09T01:41:35,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/5593f999b32f48b29e35237f114a53ee as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5593f999b32f48b29e35237f114a53ee 2024-12-09T01:41:35,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T01:41:35,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:35,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:35,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:35,754 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:35,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:35,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:35,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T01:41:35,767 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5593f999b32f48b29e35237f114a53ee, entries=300, sequenceid=63, filesize=55.4 K 2024-12-09T01:41:35,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/f00ec6e685d74a048bd6d1c13429d4e0 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/f00ec6e685d74a048bd6d1c13429d4e0 2024-12-09T01:41:35,775 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/f00ec6e685d74a048bd6d1c13429d4e0, entries=150, sequenceid=63, filesize=11.7 K 2024-12-09T01:41:35,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/e160020cd52148889093e85ee8fb8227 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/e160020cd52148889093e85ee8fb8227 2024-12-09T01:41:35,787 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/e160020cd52148889093e85ee8fb8227, entries=150, sequenceid=63, filesize=11.7 K 2024-12-09T01:41:35,797 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ca70d838c08a81d31de52aceb8d05bfd in 709ms, sequenceid=63, compaction requested=true 2024-12-09T01:41:35,797 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:35,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca70d838c08a81d31de52aceb8d05bfd:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:41:35,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:35,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca70d838c08a81d31de52aceb8d05bfd:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:41:35,797 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:35,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:35,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca70d838c08a81d31de52aceb8d05bfd:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:41:35,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T01:41:35,798 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:35,799 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:35,799 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): ca70d838c08a81d31de52aceb8d05bfd/A is initiating minor compaction (all files) 2024-12-09T01:41:35,799 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca70d838c08a81d31de52aceb8d05bfd/A in TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:35,799 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/ba7b6217a631419cbb8bba6eff13a391, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5dd60be42f644445a0e6b7e4e048121a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5593f999b32f48b29e35237f114a53ee] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp, totalSize=115.9 K 2024-12-09T01:41:35,799 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:35,800 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. files: [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/ba7b6217a631419cbb8bba6eff13a391, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5dd60be42f644445a0e6b7e4e048121a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5593f999b32f48b29e35237f114a53ee] 2024-12-09T01:41:35,800 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba7b6217a631419cbb8bba6eff13a391, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733708492669 2024-12-09T01:41:35,800 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:35,800 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): ca70d838c08a81d31de52aceb8d05bfd/B is initiating minor compaction (all files) 2024-12-09T01:41:35,801 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca70d838c08a81d31de52aceb8d05bfd/B in TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:35,801 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/5f25f9dd97e6408ca6ba863add174329, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/9ef94889f8fe418da1d7390e0898ff51, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/f00ec6e685d74a048bd6d1c13429d4e0] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp, totalSize=35.2 K 2024-12-09T01:41:35,801 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dd60be42f644445a0e6b7e4e048121a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733708492788 2024-12-09T01:41:35,801 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f25f9dd97e6408ca6ba863add174329, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733708492669 2024-12-09T01:41:35,801 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5593f999b32f48b29e35237f114a53ee, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733708495061 2024-12-09T01:41:35,802 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ef94889f8fe418da1d7390e0898ff51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733708492788 2024-12-09T01:41:35,802 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting f00ec6e685d74a048bd6d1c13429d4e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733708495085 2024-12-09T01:41:35,822 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca70d838c08a81d31de52aceb8d05bfd#B#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:35,823 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/0632f16c711940a381009f2a58c39ba5 is 50, key is test_row_0/B:col10/1733708495085/Put/seqid=0 2024-12-09T01:41:35,825 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:35,837 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412098ae794ca1ae64926a17581daccba8eed_ca70d838c08a81d31de52aceb8d05bfd store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:35,840 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412098ae794ca1ae64926a17581daccba8eed_ca70d838c08a81d31de52aceb8d05bfd, store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:35,841 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098ae794ca1ae64926a17581daccba8eed_ca70d838c08a81d31de52aceb8d05bfd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:35,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741987_1163 (size=12104) 2024-12-09T01:41:35,858 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/0632f16c711940a381009f2a58c39ba5 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/0632f16c711940a381009f2a58c39ba5 2024-12-09T01:41:35,865 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca70d838c08a81d31de52aceb8d05bfd/B of ca70d838c08a81d31de52aceb8d05bfd into 0632f16c711940a381009f2a58c39ba5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:35,865 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:35,865 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., storeName=ca70d838c08a81d31de52aceb8d05bfd/B, priority=13, startTime=1733708495797; duration=0sec 2024-12-09T01:41:35,865 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:35,865 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca70d838c08a81d31de52aceb8d05bfd:B 2024-12-09T01:41:35,866 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:35,878 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:35,878 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): ca70d838c08a81d31de52aceb8d05bfd/C is initiating minor compaction (all files) 2024-12-09T01:41:35,878 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca70d838c08a81d31de52aceb8d05bfd/C in TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:35,878 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/6f37d1eb87dc4f01ba589628f3007560, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/a88fb51330f244b795bbf1832d831013, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/e160020cd52148889093e85ee8fb8227] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp, totalSize=35.2 K 2024-12-09T01:41:35,879 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f37d1eb87dc4f01ba589628f3007560, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1733708492669 2024-12-09T01:41:35,879 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting a88fb51330f244b795bbf1832d831013, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1733708492788 2024-12-09T01:41:35,879 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting e160020cd52148889093e85ee8fb8227, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733708495085 2024-12-09T01:41:35,894 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca70d838c08a81d31de52aceb8d05bfd#C#compaction#146 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:35,895 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/624b725017ee4eca90e1c5900fd4434f is 50, key is test_row_0/C:col10/1733708495085/Put/seqid=0 2024-12-09T01:41:35,895 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733708495256, max timestamp=1733708495891Num uniques 1027; 2024-12-09T01:41:35,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741988_1164 (size=4469) 2024-12-09T01:41:35,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-09T01:41:35,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733708495256, max timestamp=1733708495891Num uniques 1027; 2024-12-09T01:41:35,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-12-09T01:41:35,902 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca70d838c08a81d31de52aceb8d05bfd 3/3 column families, dataSize=64.85 KB heapSize=177.95 KB 2024-12-09T01:41:35,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=A 2024-12-09T01:41:35,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:35,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=B 2024-12-09T01:41:35,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:35,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=C 2024-12-09T01:41:35,902 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T01:41:35,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:35,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T01:41:35,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:35,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:35,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:35,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:35,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:35,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:35,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741989_1165 (size=12104) 2024-12-09T01:41:35,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f545d190090d499eb6d138c51e963927_ca70d838c08a81d31de52aceb8d05bfd is 50, key is test_row_0/A:col10/1733708495256/Put/seqid=0 2024-12-09T01:41:35,959 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/624b725017ee4eca90e1c5900fd4434f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/624b725017ee4eca90e1c5900fd4434f 2024-12-09T01:41:35,971 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca70d838c08a81d31de52aceb8d05bfd/C of ca70d838c08a81d31de52aceb8d05bfd into 624b725017ee4eca90e1c5900fd4434f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:35,972 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:35,972 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., storeName=ca70d838c08a81d31de52aceb8d05bfd/C, priority=13, startTime=1733708495797; duration=0sec 2024-12-09T01:41:35,972 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:35,972 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca70d838c08a81d31de52aceb8d05bfd:C 2024-12-09T01:41:35,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708555976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:35,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708555979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:35,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:35,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708555981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:35,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:35,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741990_1166 (size=14594) 2024-12-09T01:41:35,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:35,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708555984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:35,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:35,996 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:35,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708555989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:35,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:35,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:36,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T01:41:36,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:36,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:36,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:36,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:36,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:36,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:36,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:36,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708556094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:36,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708556094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708556094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:36,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708556104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:36,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708556104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:36,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,112 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:36,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T01:41:36,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:36,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:36,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:36,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:36,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:36,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T01:41:36,301 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca70d838c08a81d31de52aceb8d05bfd#A#compaction#145 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:36,302 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/6d7f0f5586da4c4caac78312da9603ad is 175, key is test_row_0/A:col10/1733708495085/Put/seqid=0 2024-12-09T01:41:36,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:36,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708556303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:36,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:36,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708556314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:36,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:36,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:36,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708556314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:36,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:36,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:36,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708556314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:36,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,320 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:36,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:36,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708556314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:36,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:36,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741991_1167 (size=31058) 2024-12-09T01:41:36,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T01:41:36,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:36,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:36,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:36,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:36,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:36,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:36,394 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:36,401 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f545d190090d499eb6d138c51e963927_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f545d190090d499eb6d138c51e963927_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:36,403 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/5e8a9207ef0f419eb37279db59135c52, store: [table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:36,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/5e8a9207ef0f419eb37279db59135c52 is 175, key is test_row_0/A:col10/1733708495256/Put/seqid=0 2024-12-09T01:41:36,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741992_1168 (size=39549) 2024-12-09T01:41:36,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T01:41:36,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:36,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:36,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:36,533 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:36,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:36,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708556616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:36,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708556624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,632 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:36,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708556624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:36,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708556624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:36,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708556634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:36,640 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:36,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T01:41:36,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:36,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:36,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:36,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:36,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:36,756 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/6d7f0f5586da4c4caac78312da9603ad as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/6d7f0f5586da4c4caac78312da9603ad 2024-12-09T01:41:36,762 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca70d838c08a81d31de52aceb8d05bfd/A of ca70d838c08a81d31de52aceb8d05bfd into 6d7f0f5586da4c4caac78312da9603ad(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:36,762 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:36,762 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., storeName=ca70d838c08a81d31de52aceb8d05bfd/A, priority=13, startTime=1733708495797; duration=0sec 2024-12-09T01:41:36,762 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:36,762 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca70d838c08a81d31de52aceb8d05bfd:A 2024-12-09T01:41:36,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T01:41:36,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:36,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:36,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:36,841 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:36,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:36,856 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/5e8a9207ef0f419eb37279db59135c52 2024-12-09T01:41:36,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/9fbb60bc2e2543cc8fbc1205c6ed7032 is 50, key is test_row_0/B:col10/1733708495256/Put/seqid=0 2024-12-09T01:41:36,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741993_1169 (size=12001) 2024-12-09T01:41:36,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T01:41:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:36,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:36,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:36,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:37,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:37,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708557123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:37,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:37,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:37,133 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:37,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T01:41:37,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:37,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:37,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:37,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:37,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:37,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708557144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:37,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708557144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:37,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708557144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:37,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708557144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:37,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:37,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:37,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T01:41:37,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T01:41:37,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:37,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:37,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:37,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:37,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:37,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:37,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/9fbb60bc2e2543cc8fbc1205c6ed7032 2024-12-09T01:41:37,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/067515fc844d4d7cada2f9ec351bf953 is 50, key is test_row_0/C:col10/1733708495256/Put/seqid=0 2024-12-09T01:41:37,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741994_1170 (size=12001) 2024-12-09T01:41:37,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/067515fc844d4d7cada2f9ec351bf953 2024-12-09T01:41:37,413 INFO [master/ef6f18c58dc9:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-09T01:41:37,413 INFO [master/ef6f18c58dc9:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-09T01:41:37,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/5e8a9207ef0f419eb37279db59135c52 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5e8a9207ef0f419eb37279db59135c52 2024-12-09T01:41:37,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5e8a9207ef0f419eb37279db59135c52, entries=200, sequenceid=119, filesize=38.6 K 2024-12-09T01:41:37,427 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/9fbb60bc2e2543cc8fbc1205c6ed7032 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/9fbb60bc2e2543cc8fbc1205c6ed7032 2024-12-09T01:41:37,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/9fbb60bc2e2543cc8fbc1205c6ed7032, entries=150, sequenceid=119, filesize=11.7 K 2024-12-09T01:41:37,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/067515fc844d4d7cada2f9ec351bf953 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/067515fc844d4d7cada2f9ec351bf953 2024-12-09T01:41:37,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/067515fc844d4d7cada2f9ec351bf953, entries=150, sequenceid=119, filesize=11.7 K 2024-12-09T01:41:37,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~64.85 KB/66410, heapSize ~177.91 KB/182176, currentSize=114.05 KB/116790 for ca70d838c08a81d31de52aceb8d05bfd in 1542ms, sequenceid=119, compaction requested=false 2024-12-09T01:41:37,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:37,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-12-09T01:41:37,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:37,459 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing ca70d838c08a81d31de52aceb8d05bfd 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T01:41:37,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=A 2024-12-09T01:41:37,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:37,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=B 2024-12-09T01:41:37,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:37,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=C 2024-12-09T01:41:37,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:37,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209215ece1774e64be1a6750290d7557c9e_ca70d838c08a81d31de52aceb8d05bfd is 50, key is test_row_0/A:col10/1733708495987/Put/seqid=0 2024-12-09T01:41:37,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741995_1171 (size=12304) 2024-12-09T01:41:37,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,541 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209215ece1774e64be1a6750290d7557c9e_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209215ece1774e64be1a6750290d7557c9e_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:37,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/2218d6fe5a074f2bb00d26523c6070b1, store: [table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:37,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/2218d6fe5a074f2bb00d26523c6070b1 is 175, key is test_row_0/A:col10/1733708495987/Put/seqid=0 2024-12-09T01:41:37,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741996_1172 (size=31105) 2024-12-09T01:41:37,587 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=141, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/2218d6fe5a074f2bb00d26523c6070b1 2024-12-09T01:41:37,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/f107f57fcfec48c1bbda31d721709311 is 50, key is test_row_0/B:col10/1733708495987/Put/seqid=0 2024-12-09T01:41:37,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741997_1173 (size=12151) 2024-12-09T01:41:37,613 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/f107f57fcfec48c1bbda31d721709311 2024-12-09T01:41:37,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/46b10ff016ec43e992e4fc0e882d00b4 is 50, key is test_row_0/C:col10/1733708495987/Put/seqid=0 2024-12-09T01:41:37,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741998_1174 (size=12151) 2024-12-09T01:41:37,633 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/46b10ff016ec43e992e4fc0e882d00b4 2024-12-09T01:41:37,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/2218d6fe5a074f2bb00d26523c6070b1 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/2218d6fe5a074f2bb00d26523c6070b1 2024-12-09T01:41:37,661 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/2218d6fe5a074f2bb00d26523c6070b1, entries=150, sequenceid=141, filesize=30.4 K 2024-12-09T01:41:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/f107f57fcfec48c1bbda31d721709311 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/f107f57fcfec48c1bbda31d721709311 2024-12-09T01:41:37,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,673 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/f107f57fcfec48c1bbda31d721709311, entries=150, sequenceid=141, filesize=11.9 K 2024-12-09T01:41:37,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/46b10ff016ec43e992e4fc0e882d00b4 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/46b10ff016ec43e992e4fc0e882d00b4 2024-12-09T01:41:37,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,681 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/46b10ff016ec43e992e4fc0e882d00b4, entries=150, sequenceid=141, filesize=11.9 K 2024-12-09T01:41:37,683 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for ca70d838c08a81d31de52aceb8d05bfd in 224ms, sequenceid=141, compaction requested=true 2024-12-09T01:41:37,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:37,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:37,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-12-09T01:41:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-12-09T01:41:37,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-12-09T01:41:37,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5560 sec 2024-12-09T01:41:37,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.5640 sec 2024-12-09T01:41:37,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:38,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca70d838c08a81d31de52aceb8d05bfd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:41:38,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=A 2024-12-09T01:41:38,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:38,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=B 2024-12-09T01:41:38,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:38,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=C 2024-12-09T01:41:38,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:38,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,239 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098a5010b89c1b45aea67369497ac8a664_ca70d838c08a81d31de52aceb8d05bfd is 50, key is test_row_0/A:col10/1733708498208/Put/seqid=0 2024-12-09T01:41:38,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1733708498247, max timestamp=1733708498263Num uniques 859; 2024-12-09T01:41:38,267 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:41:38,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708498247, max timestamp=1733708498263Num uniques 859; 2024-12-09T01:41:38,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:41:38,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708498247, max timestamp=1733708498263Num uniques 859; 2024-12-09T01:41:38,274 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:41:38,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741999_1175 (size=22268) 2024-12-09T01:41:38,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733708498290, max timestamp=1733708498336Num uniques 1068; 2024-12-09T01:41:38,341 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-09T01:41:38,344 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733708498290, max timestamp=1733708498336Num uniques 1068; 2024-12-09T01:41:38,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-09T01:41:38,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733708498290, max timestamp=1733708498336Num uniques 1068; 2024-12-09T01:41:38,345 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=6870, suffix heap size=139040,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=150 2024-12-09T01:41:38,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:38,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708558379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:38,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708558379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:38,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708558381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:38,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:38,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708558382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:38,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:38,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:38,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708558398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:38,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,403 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:38,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:38,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708558504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:38,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:38,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708558504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:38,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:38,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,507 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:38,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:38,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708558506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:38,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:38,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708558514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:38,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:38,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,523 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:38,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708558514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:38,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:38,687 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:38,693 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098a5010b89c1b45aea67369497ac8a664_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098a5010b89c1b45aea67369497ac8a664_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:38,694 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/e46076ed4ac24e69b86ff22df55c85de, store: [table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:38,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/e46076ed4ac24e69b86ff22df55c85de is 175, key is test_row_0/A:col10/1733708498208/Put/seqid=0 2024-12-09T01:41:38,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708558714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:38,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:38,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708558714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:38,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:38,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708558714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:38,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,721 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:38,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742000_1176 (size=65673) 2024-12-09T01:41:38,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:38,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:38,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708558734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:38,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708558734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:38,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:38,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:38,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:39,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:39,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708559024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:39,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:39,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:39,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708559024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:39,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708559024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,037 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:39,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:39,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708559054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:39,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,063 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:39,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:39,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708559063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,066 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:39,129 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/e46076ed4ac24e69b86ff22df55c85de 2024-12-09T01:41:39,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/42605c26f12a4007a7c2780b5607b10d is 50, key is test_row_0/B:col10/1733708498208/Put/seqid=0 2024-12-09T01:41:39,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742001_1177 (size=12151) 2024-12-09T01:41:39,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/42605c26f12a4007a7c2780b5607b10d 2024-12-09T01:41:39,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/a1af3b6d6c5a4d5d8f86e063a1568f82 is 50, key is test_row_0/C:col10/1733708498208/Put/seqid=0 2024-12-09T01:41:39,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742002_1178 (size=12151) 2024-12-09T01:41:39,192 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/a1af3b6d6c5a4d5d8f86e063a1568f82 2024-12-09T01:41:39,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/e46076ed4ac24e69b86ff22df55c85de as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/e46076ed4ac24e69b86ff22df55c85de 2024-12-09T01:41:39,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/e46076ed4ac24e69b86ff22df55c85de, entries=350, sequenceid=151, filesize=64.1 K 2024-12-09T01:41:39,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/42605c26f12a4007a7c2780b5607b10d as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/42605c26f12a4007a7c2780b5607b10d 2024-12-09T01:41:39,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/42605c26f12a4007a7c2780b5607b10d, entries=150, sequenceid=151, filesize=11.9 K 2024-12-09T01:41:39,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/a1af3b6d6c5a4d5d8f86e063a1568f82 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/a1af3b6d6c5a4d5d8f86e063a1568f82 2024-12-09T01:41:39,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/a1af3b6d6c5a4d5d8f86e063a1568f82, entries=150, sequenceid=151, filesize=11.9 K 2024-12-09T01:41:39,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=135.62 KB/138872 for ca70d838c08a81d31de52aceb8d05bfd in 1000ms, sequenceid=151, compaction requested=true 2024-12-09T01:41:39,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:39,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca70d838c08a81d31de52aceb8d05bfd:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:41:39,222 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:41:39,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:39,222 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:41:39,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca70d838c08a81d31de52aceb8d05bfd:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:41:39,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:39,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca70d838c08a81d31de52aceb8d05bfd:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:41:39,222 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:39,225 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48407 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:41:39,226 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): ca70d838c08a81d31de52aceb8d05bfd/B is initiating minor compaction (all files) 2024-12-09T01:41:39,226 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca70d838c08a81d31de52aceb8d05bfd/B in TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:39,226 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/0632f16c711940a381009f2a58c39ba5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/9fbb60bc2e2543cc8fbc1205c6ed7032, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/f107f57fcfec48c1bbda31d721709311, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/42605c26f12a4007a7c2780b5607b10d] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp, totalSize=47.3 K 2024-12-09T01:41:39,226 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 167385 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:41:39,226 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): ca70d838c08a81d31de52aceb8d05bfd/A is initiating minor compaction (all files) 2024-12-09T01:41:39,226 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca70d838c08a81d31de52aceb8d05bfd/A in TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:39,227 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/6d7f0f5586da4c4caac78312da9603ad, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5e8a9207ef0f419eb37279db59135c52, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/2218d6fe5a074f2bb00d26523c6070b1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/e46076ed4ac24e69b86ff22df55c85de] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp, totalSize=163.5 K 2024-12-09T01:41:39,227 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:39,227 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. files: [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/6d7f0f5586da4c4caac78312da9603ad, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5e8a9207ef0f419eb37279db59135c52, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/2218d6fe5a074f2bb00d26523c6070b1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/e46076ed4ac24e69b86ff22df55c85de] 2024-12-09T01:41:39,227 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d7f0f5586da4c4caac78312da9603ad, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733708495085 2024-12-09T01:41:39,228 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e8a9207ef0f419eb37279db59135c52, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733708495256 2024-12-09T01:41:39,228 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 0632f16c711940a381009f2a58c39ba5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733708495085 2024-12-09T01:41:39,229 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2218d6fe5a074f2bb00d26523c6070b1, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733708495963 2024-12-09T01:41:39,229 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fbb60bc2e2543cc8fbc1205c6ed7032, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733708495256 2024-12-09T01:41:39,230 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting e46076ed4ac24e69b86ff22df55c85de, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733708498170 2024-12-09T01:41:39,230 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting f107f57fcfec48c1bbda31d721709311, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733708495963 2024-12-09T01:41:39,231 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 42605c26f12a4007a7c2780b5607b10d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733708498208 2024-12-09T01:41:39,243 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca70d838c08a81d31de52aceb8d05bfd#B#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:39,243 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/25cb591e9ccb488a8e8e39e94a34deff is 50, key is test_row_0/B:col10/1733708498208/Put/seqid=0 2024-12-09T01:41:39,244 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:39,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742003_1179 (size=12391) 2024-12-09T01:41:39,256 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120958b49d0c64de4bf9a4b5dc8e016be0db_ca70d838c08a81d31de52aceb8d05bfd store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:39,260 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120958b49d0c64de4bf9a4b5dc8e016be0db_ca70d838c08a81d31de52aceb8d05bfd, store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:39,260 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120958b49d0c64de4bf9a4b5dc8e016be0db_ca70d838c08a81d31de52aceb8d05bfd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:39,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742004_1180 (size=4469) 2024-12-09T01:41:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-12-09T01:41:39,294 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:41:39,294 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:41:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-12-09T01:41:39,297 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:41:39,297 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:41:39,297 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:41:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T01:41:39,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T01:41:39,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-12-09T01:41:39,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:39,451 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing ca70d838c08a81d31de52aceb8d05bfd 3/3 column families, dataSize=135.62 KB heapSize=382.70 KB 2024-12-09T01:41:39,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=A 2024-12-09T01:41:39,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:39,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=B 2024-12-09T01:41:39,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:39,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=C 2024-12-09T01:41:39,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:39,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209898ca7079ef64a7cac2184427b88cb33_ca70d838c08a81d31de52aceb8d05bfd is 50, key is test_row_0/A:col10/1733708498396/Put/seqid=0 2024-12-09T01:41:39,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742005_1181 (size=12304) 2024-12-09T01:41:39,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:39,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:39,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:39,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708559560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:39,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:39,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708559562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,565 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,566 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:39,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:39,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:39,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708559574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:39,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708559574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:39,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,583 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:39,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708559565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:39,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,584 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:39,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T01:41:39,664 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/25cb591e9ccb488a8e8e39e94a34deff as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/25cb591e9ccb488a8e8e39e94a34deff 2024-12-09T01:41:39,672 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca70d838c08a81d31de52aceb8d05bfd/B of ca70d838c08a81d31de52aceb8d05bfd into 25cb591e9ccb488a8e8e39e94a34deff(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:39,672 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:39,672 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., storeName=ca70d838c08a81d31de52aceb8d05bfd/B, priority=12, startTime=1733708499222; duration=0sec 2024-12-09T01:41:39,673 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:39,673 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca70d838c08a81d31de52aceb8d05bfd:B 2024-12-09T01:41:39,673 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:41:39,676 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca70d838c08a81d31de52aceb8d05bfd#A#compaction#157 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:39,677 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/3dd7d191973c4e0586b71f301a71ad76 is 175, key is test_row_0/A:col10/1733708498208/Put/seqid=0 2024-12-09T01:41:39,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708559674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:39,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708559674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:39,681 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48407 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:41:39,681 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): ca70d838c08a81d31de52aceb8d05bfd/C is initiating minor compaction (all files) 2024-12-09T01:41:39,681 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca70d838c08a81d31de52aceb8d05bfd/C in TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:39,681 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/624b725017ee4eca90e1c5900fd4434f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/067515fc844d4d7cada2f9ec351bf953, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/46b10ff016ec43e992e4fc0e882d00b4, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/a1af3b6d6c5a4d5d8f86e063a1568f82] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp, totalSize=47.3 K 2024-12-09T01:41:39,682 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 624b725017ee4eca90e1c5900fd4434f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1733708495085 2024-12-09T01:41:39,682 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 067515fc844d4d7cada2f9ec351bf953, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1733708495256 2024-12-09T01:41:39,683 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 46b10ff016ec43e992e4fc0e882d00b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1733708495963 2024-12-09T01:41:39,683 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting a1af3b6d6c5a4d5d8f86e063a1568f82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733708498208 2024-12-09T01:41:39,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:39,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708559693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:39,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,697 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:39,701 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca70d838c08a81d31de52aceb8d05bfd#C#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:39,702 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/d26094c2aefb498eb0ccfa5ee346227a is 50, key is test_row_0/C:col10/1733708498208/Put/seqid=0 2024-12-09T01:41:39,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742006_1182 (size=31345) 2024-12-09T01:41:39,720 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/3dd7d191973c4e0586b71f301a71ad76 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/3dd7d191973c4e0586b71f301a71ad76 2024-12-09T01:41:39,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742007_1183 (size=12391) 2024-12-09T01:41:39,727 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca70d838c08a81d31de52aceb8d05bfd/A of ca70d838c08a81d31de52aceb8d05bfd into 3dd7d191973c4e0586b71f301a71ad76(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:39,727 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:39,727 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., storeName=ca70d838c08a81d31de52aceb8d05bfd/A, priority=12, startTime=1733708499221; duration=0sec 2024-12-09T01:41:39,727 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:39,727 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca70d838c08a81d31de52aceb8d05bfd:A 2024-12-09T01:41:39,737 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/d26094c2aefb498eb0ccfa5ee346227a as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/d26094c2aefb498eb0ccfa5ee346227a 2024-12-09T01:41:39,744 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca70d838c08a81d31de52aceb8d05bfd/C of ca70d838c08a81d31de52aceb8d05bfd into d26094c2aefb498eb0ccfa5ee346227a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:39,744 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:39,745 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., storeName=ca70d838c08a81d31de52aceb8d05bfd/C, priority=12, startTime=1733708499222; duration=0sec 2024-12-09T01:41:39,745 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:39,745 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca70d838c08a81d31de52aceb8d05bfd:C 2024-12-09T01:41:39,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:39,874 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209898ca7079ef64a7cac2184427b88cb33_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209898ca7079ef64a7cac2184427b88cb33_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:39,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/06cfac23d163446381785a5ab72885ee, store: [table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:39,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/06cfac23d163446381785a5ab72885ee is 175, key is test_row_0/A:col10/1733708498396/Put/seqid=0 2024-12-09T01:41:39,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742008_1184 (size=31105) 2024-12-09T01:41:39,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708559883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:39,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:39,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708559884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:39,888 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:39,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708559903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:39,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T01:41:40,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708560193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:40,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:40,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708560194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:40,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:40,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:40,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:40,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:40,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708560213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:40,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:40,215 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:40,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:40,282 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/06cfac23d163446381785a5ab72885ee 2024-12-09T01:41:40,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/8e49142163884babaf6c347f74a150c0 is 50, key is test_row_0/B:col10/1733708498396/Put/seqid=0 2024-12-09T01:41:40,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742009_1185 (size=12151) 2024-12-09T01:41:40,334 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/8e49142163884babaf6c347f74a150c0 2024-12-09T01:41:40,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/d6b41cafe8ff4ddbabe30b2409988408 is 50, key is test_row_0/C:col10/1733708498396/Put/seqid=0 2024-12-09T01:41:40,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742010_1186 (size=12151) 2024-12-09T01:41:40,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T01:41:40,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708560594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:40,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708560594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:40,602 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_1' in ca70d838c08a81d31de52aceb8d05bfd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:41:40,602 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_1' in ca70d838c08a81d31de52aceb8d05bfd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:41:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:40,603 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:40,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708560703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:40,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708560703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:40,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:40,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:40,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708560723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:40,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:40,764 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.69 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/d6b41cafe8ff4ddbabe30b2409988408 2024-12-09T01:41:40,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/06cfac23d163446381785a5ab72885ee as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/06cfac23d163446381785a5ab72885ee 2024-12-09T01:41:40,776 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/06cfac23d163446381785a5ab72885ee, entries=150, sequenceid=204, filesize=30.4 K 2024-12-09T01:41:40,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/8e49142163884babaf6c347f74a150c0 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/8e49142163884babaf6c347f74a150c0 2024-12-09T01:41:40,782 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/8e49142163884babaf6c347f74a150c0, entries=150, sequenceid=204, filesize=11.9 K 2024-12-09T01:41:40,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/d6b41cafe8ff4ddbabe30b2409988408 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/d6b41cafe8ff4ddbabe30b2409988408 2024-12-09T01:41:40,787 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/d6b41cafe8ff4ddbabe30b2409988408, entries=150, sequenceid=204, filesize=11.9 K 2024-12-09T01:41:40,789 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~135.62 KB/138872, heapSize ~382.66 KB/391840, currentSize=46.96 KB/48090 for ca70d838c08a81d31de52aceb8d05bfd in 1337ms, sequenceid=204, compaction requested=false 2024-12-09T01:41:40,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:40,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:40,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-12-09T01:41:40,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-12-09T01:41:40,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-12-09T01:41:40,793 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4930 sec 2024-12-09T01:41:40,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.5000 sec 2024-12-09T01:41:41,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-12-09T01:41:41,444 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:41:41,445 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:41:41,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-12-09T01:41:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T01:41:41,447 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:41:41,448 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:41:41,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:41:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T01:41:41,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-12-09T01:41:41,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:41,601 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing ca70d838c08a81d31de52aceb8d05bfd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:41:41,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=A 2024-12-09T01:41:41,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:41,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=B 2024-12-09T01:41:41,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:41,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=C 2024-12-09T01:41:41,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:41,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095c529ca6567b447992673e53c5a89fd1_ca70d838c08a81d31de52aceb8d05bfd is 50, key is test_row_0/A:col10/1733708499561/Put/seqid=0 2024-12-09T01:41:41,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742011_1187 (size=12304) 2024-12-09T01:41:41,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:41,625 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412095c529ca6567b447992673e53c5a89fd1_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095c529ca6567b447992673e53c5a89fd1_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:41,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/8ebaf5de9e13425d8ed67de66428e5f6, store: [table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:41,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/8ebaf5de9e13425d8ed67de66428e5f6 is 175, key is test_row_0/A:col10/1733708499561/Put/seqid=0 2024-12-09T01:41:41,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742012_1188 (size=31105) 2024-12-09T01:41:41,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:41,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:41,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T01:41:41,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733708501737, max timestamp=1733708501769Num uniques 859; 2024-12-09T01:41:41,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-09T01:41:41,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733708501737, max timestamp=1733708501769Num uniques 859; 2024-12-09T01:41:41,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-09T01:41:41,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1733708501737, max timestamp=1733708501769Num uniques 859; 2024-12-09T01:41:41,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-12-09T01:41:41,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:41,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708561803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:41,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:41,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:41,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708561804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:41,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:41,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:41,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:41,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:41,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:41,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:41,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708561806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:41,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:41,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:41,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:41,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:41,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708561913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:41,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:41,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:41,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708561913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:41,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:41,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708561913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:41,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:42,039 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=217, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/8ebaf5de9e13425d8ed67de66428e5f6 2024-12-09T01:41:42,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/81ea69653e9d4cbfa17ca12d019101c1 is 50, key is test_row_0/B:col10/1733708499561/Put/seqid=0 2024-12-09T01:41:42,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742013_1189 (size=12151) 2024-12-09T01:41:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T01:41:42,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708562123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:42,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:42,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:42,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708562124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:42,128 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:42,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:42,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:42,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:42,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:42,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708562124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:42,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:42,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:42,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:42,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:42,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708562433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:42,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:42,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708562433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:42,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:42,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:42,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:42,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:42,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:42,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:42,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708562433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:42,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:42,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:42,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:42,455 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/81ea69653e9d4cbfa17ca12d019101c1 2024-12-09T01:41:42,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/bacfbd84e03141c8b3b29693f4592e0b is 50, key is test_row_0/C:col10/1733708499561/Put/seqid=0 2024-12-09T01:41:42,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742014_1190 (size=12151) 2024-12-09T01:41:42,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T01:41:42,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:42,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708562624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:42,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:42,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708562624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:42,627 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_1' in ca70d838c08a81d31de52aceb8d05bfd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:41:42,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:42,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:42,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:42,628 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_1' in ca70d838c08a81d31de52aceb8d05bfd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4247 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:41:42,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:42,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:42,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:42,876 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/bacfbd84e03141c8b3b29693f4592e0b 2024-12-09T01:41:42,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/8ebaf5de9e13425d8ed67de66428e5f6 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/8ebaf5de9e13425d8ed67de66428e5f6 2024-12-09T01:41:42,889 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/8ebaf5de9e13425d8ed67de66428e5f6, entries=150, sequenceid=217, filesize=30.4 K 2024-12-09T01:41:42,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/81ea69653e9d4cbfa17ca12d019101c1 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/81ea69653e9d4cbfa17ca12d019101c1 2024-12-09T01:41:42,895 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/81ea69653e9d4cbfa17ca12d019101c1, entries=150, sequenceid=217, filesize=11.9 K 2024-12-09T01:41:42,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/bacfbd84e03141c8b3b29693f4592e0b as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/bacfbd84e03141c8b3b29693f4592e0b 2024-12-09T01:41:42,904 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/bacfbd84e03141c8b3b29693f4592e0b, entries=150, sequenceid=217, filesize=11.9 K 2024-12-09T01:41:42,905 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for ca70d838c08a81d31de52aceb8d05bfd in 1304ms, sequenceid=217, compaction requested=true 2024-12-09T01:41:42,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:42,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:42,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-12-09T01:41:42,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-12-09T01:41:42,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-12-09T01:41:42,909 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4590 sec 2024-12-09T01:41:42,911 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.4650 sec 2024-12-09T01:41:42,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:42,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca70d838c08a81d31de52aceb8d05bfd 3/3 column families, dataSize=147.60 KB heapSize=401.39 KB 2024-12-09T01:41:42,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=A 2024-12-09T01:41:42,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:42,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=B 2024-12-09T01:41:42,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:42,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=C 2024-12-09T01:41:42,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:42,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120972f078f20e6c40179f3204833c29dd2b_ca70d838c08a81d31de52aceb8d05bfd is 50, key is test_row_0/A:col10/1733708501804/Put/seqid=0 2024-12-09T01:41:42,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742015_1191 (size=14794) 2024-12-09T01:41:42,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708562958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:42,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:42,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:42,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708562959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:42,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:42,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:42,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708562960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:42,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:42,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:42,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:42,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:43,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:43,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708563073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:43,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:43,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:43,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:43,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:43,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708563073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:43,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:43,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:43,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:43,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:43,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708563074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:43,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:43,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:43,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:43,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:43,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708563283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:43,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:43,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708563283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:43,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:43,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:43,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708563283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:43,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:43,362 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:43,371 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120972f078f20e6c40179f3204833c29dd2b_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120972f078f20e6c40179f3204833c29dd2b_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:43,372 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/965c73ab899e4ff8b8cee96d305300db, store: [table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:43,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/965c73ab899e4ff8b8cee96d305300db is 175, key is test_row_0/A:col10/1733708501804/Put/seqid=0 2024-12-09T01:41:43,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742016_1192 (size=39749) 2024-12-09T01:41:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-09T01:41:43,594 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:41:43,594 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:41:43,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-12-09T01:41:43,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T01:41:43,596 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:41:43,597 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:41:43,597 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:41:43,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:43,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708563604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:43,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:43,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708563604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:43,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708563604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:43,607 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:43,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T01:41:43,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-09T01:41:43,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:43,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:43,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:43,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:43,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:43,802 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/965c73ab899e4ff8b8cee96d305300db 2024-12-09T01:41:43,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/1990232f30f148e6bb0b078597f772bf is 50, key is test_row_0/B:col10/1733708501804/Put/seqid=0 2024-12-09T01:41:43,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742017_1193 (size=12151) 2024-12-09T01:41:43,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-09T01:41:43,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:43,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:43,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:43,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:43,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:43,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:43,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T01:41:44,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-09T01:41:44,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:44,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:44,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:44,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:44,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:44,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:44,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:44,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708564114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:44,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:44,117 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708564115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:44,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708564123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:44,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:44,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-09T01:41:44,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:44,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:44,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:44,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:44,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:44,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:44,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/1990232f30f148e6bb0b078597f772bf 2024-12-09T01:41:44,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T01:41:44,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/756da9c587054ffdbb6b6dcb397b9928 is 50, key is test_row_0/C:col10/1733708501804/Put/seqid=0 2024-12-09T01:41:44,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742018_1194 (size=12151) 2024-12-09T01:41:44,232 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/756da9c587054ffdbb6b6dcb397b9928 2024-12-09T01:41:44,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/965c73ab899e4ff8b8cee96d305300db as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/965c73ab899e4ff8b8cee96d305300db 2024-12-09T01:41:44,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/965c73ab899e4ff8b8cee96d305300db, entries=200, sequenceid=251, filesize=38.8 K 2024-12-09T01:41:44,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/1990232f30f148e6bb0b078597f772bf as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/1990232f30f148e6bb0b078597f772bf 2024-12-09T01:41:44,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/1990232f30f148e6bb0b078597f772bf, entries=150, sequenceid=251, filesize=11.9 K 2024-12-09T01:41:44,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/756da9c587054ffdbb6b6dcb397b9928 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/756da9c587054ffdbb6b6dcb397b9928 2024-12-09T01:41:44,257 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/756da9c587054ffdbb6b6dcb397b9928, entries=150, sequenceid=251, filesize=11.9 K 2024-12-09T01:41:44,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~401.34 KB/410976, currentSize=40.25 KB/41220 for ca70d838c08a81d31de52aceb8d05bfd in 1311ms, sequenceid=251, compaction requested=true 2024-12-09T01:41:44,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:44,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca70d838c08a81d31de52aceb8d05bfd:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:41:44,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:44,258 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:41:44,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca70d838c08a81d31de52aceb8d05bfd:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:41:44,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:44,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca70d838c08a81d31de52aceb8d05bfd:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:41:44,258 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:41:44,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:44,260 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:41:44,260 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133304 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:41:44,260 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): ca70d838c08a81d31de52aceb8d05bfd/B is initiating minor compaction (all files) 2024-12-09T01:41:44,260 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): ca70d838c08a81d31de52aceb8d05bfd/A is initiating minor compaction (all files) 2024-12-09T01:41:44,261 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca70d838c08a81d31de52aceb8d05bfd/A in TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:44,261 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca70d838c08a81d31de52aceb8d05bfd/B in TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:44,261 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/3dd7d191973c4e0586b71f301a71ad76, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/06cfac23d163446381785a5ab72885ee, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/8ebaf5de9e13425d8ed67de66428e5f6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/965c73ab899e4ff8b8cee96d305300db] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp, totalSize=130.2 K 2024-12-09T01:41:44,261 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/25cb591e9ccb488a8e8e39e94a34deff, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/8e49142163884babaf6c347f74a150c0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/81ea69653e9d4cbfa17ca12d019101c1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/1990232f30f148e6bb0b078597f772bf] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp, totalSize=47.7 K 2024-12-09T01:41:44,261 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:44,261 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. files: [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/3dd7d191973c4e0586b71f301a71ad76, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/06cfac23d163446381785a5ab72885ee, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/8ebaf5de9e13425d8ed67de66428e5f6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/965c73ab899e4ff8b8cee96d305300db] 2024-12-09T01:41:44,262 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dd7d191973c4e0586b71f301a71ad76, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733708498208 2024-12-09T01:41:44,262 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 25cb591e9ccb488a8e8e39e94a34deff, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733708498208 2024-12-09T01:41:44,262 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06cfac23d163446381785a5ab72885ee, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733708498375 2024-12-09T01:41:44,262 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e49142163884babaf6c347f74a150c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733708498375 2024-12-09T01:41:44,263 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 81ea69653e9d4cbfa17ca12d019101c1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733708499551 2024-12-09T01:41:44,263 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ebaf5de9e13425d8ed67de66428e5f6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733708499551 2024-12-09T01:41:44,263 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 1990232f30f148e6bb0b078597f772bf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733708501788 2024-12-09T01:41:44,263 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 965c73ab899e4ff8b8cee96d305300db, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733708501788 2024-12-09T01:41:44,273 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:44,275 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca70d838c08a81d31de52aceb8d05bfd#B#compaction#169 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:44,276 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/7987c2d0270644639ea9af37de2a07d9 is 50, key is test_row_0/B:col10/1733708501804/Put/seqid=0 2024-12-09T01:41:44,280 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209615336f94a944b69b6fa3c3a9f7d3a69_ca70d838c08a81d31de52aceb8d05bfd store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:44,283 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209615336f94a944b69b6fa3c3a9f7d3a69_ca70d838c08a81d31de52aceb8d05bfd, store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:44,283 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209615336f94a944b69b6fa3c3a9f7d3a69_ca70d838c08a81d31de52aceb8d05bfd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:44,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742020_1196 (size=4469) 2024-12-09T01:41:44,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742019_1195 (size=12527) 2024-12-09T01:41:44,310 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca70d838c08a81d31de52aceb8d05bfd#A#compaction#168 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:44,310 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/bcde21feb8304a2a849edc29a77caa78 is 175, key is test_row_0/A:col10/1733708501804/Put/seqid=0 2024-12-09T01:41:44,320 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/7987c2d0270644639ea9af37de2a07d9 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/7987c2d0270644639ea9af37de2a07d9 2024-12-09T01:41:44,330 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca70d838c08a81d31de52aceb8d05bfd/B of ca70d838c08a81d31de52aceb8d05bfd into 7987c2d0270644639ea9af37de2a07d9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:44,330 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:44,330 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., storeName=ca70d838c08a81d31de52aceb8d05bfd/B, priority=12, startTime=1733708504258; duration=0sec 2024-12-09T01:41:44,330 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:44,330 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca70d838c08a81d31de52aceb8d05bfd:B 2024-12-09T01:41:44,330 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:41:44,335 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:41:44,336 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): ca70d838c08a81d31de52aceb8d05bfd/C is initiating minor compaction (all files) 2024-12-09T01:41:44,336 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca70d838c08a81d31de52aceb8d05bfd/C in TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:44,337 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/d26094c2aefb498eb0ccfa5ee346227a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/d6b41cafe8ff4ddbabe30b2409988408, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/bacfbd84e03141c8b3b29693f4592e0b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/756da9c587054ffdbb6b6dcb397b9928] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp, totalSize=47.7 K 2024-12-09T01:41:44,337 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting d26094c2aefb498eb0ccfa5ee346227a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1733708498208 2024-12-09T01:41:44,338 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting d6b41cafe8ff4ddbabe30b2409988408, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733708498375 2024-12-09T01:41:44,338 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting bacfbd84e03141c8b3b29693f4592e0b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1733708499551 2024-12-09T01:41:44,338 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 756da9c587054ffdbb6b6dcb397b9928, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733708501788 2024-12-09T01:41:44,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742021_1197 (size=31481) 2024-12-09T01:41:44,348 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/bcde21feb8304a2a849edc29a77caa78 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/bcde21feb8304a2a849edc29a77caa78 2024-12-09T01:41:44,355 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca70d838c08a81d31de52aceb8d05bfd/A of ca70d838c08a81d31de52aceb8d05bfd into bcde21feb8304a2a849edc29a77caa78(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:44,356 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:44,356 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., storeName=ca70d838c08a81d31de52aceb8d05bfd/A, priority=12, startTime=1733708504258; duration=0sec 2024-12-09T01:41:44,356 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:44,356 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca70d838c08a81d31de52aceb8d05bfd:A 2024-12-09T01:41:44,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-12-09T01:41:44,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:44,365 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing ca70d838c08a81d31de52aceb8d05bfd 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-09T01:41:44,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=A 2024-12-09T01:41:44,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:44,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=B 2024-12-09T01:41:44,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:44,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=C 2024-12-09T01:41:44,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:44,389 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca70d838c08a81d31de52aceb8d05bfd#C#compaction#170 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:44,390 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/0baaa9f8e0df44fa87619e1274b3901e is 50, key is test_row_0/C:col10/1733708501804/Put/seqid=0 2024-12-09T01:41:44,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f546b7804a724b21a1d4c67f0b6b478e_ca70d838c08a81d31de52aceb8d05bfd is 50, key is test_row_0/A:col10/1733708502958/Put/seqid=0 2024-12-09T01:41:44,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742022_1198 (size=12527) 2024-12-09T01:41:44,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742023_1199 (size=12404) 2024-12-09T01:41:44,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T01:41:44,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:44,804 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/0baaa9f8e0df44fa87619e1274b3901e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/0baaa9f8e0df44fa87619e1274b3901e 2024-12-09T01:41:44,808 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f546b7804a724b21a1d4c67f0b6b478e_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f546b7804a724b21a1d4c67f0b6b478e_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:44,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/dfc75619368649b8a3f5dc617e5a2d0a, store: [table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:44,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/dfc75619368649b8a3f5dc617e5a2d0a is 175, key is test_row_0/A:col10/1733708502958/Put/seqid=0 2024-12-09T01:41:44,832 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ca70d838c08a81d31de52aceb8d05bfd/C of ca70d838c08a81d31de52aceb8d05bfd into 0baaa9f8e0df44fa87619e1274b3901e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:44,832 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:44,832 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., storeName=ca70d838c08a81d31de52aceb8d05bfd/C, priority=12, startTime=1733708504258; duration=0sec 2024-12-09T01:41:44,832 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:44,832 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca70d838c08a81d31de52aceb8d05bfd:C 2024-12-09T01:41:44,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742024_1200 (size=31205) 2024-12-09T01:41:45,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:45,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:45,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708505147, max timestamp=1733708505174Num uniques 859; 2024-12-09T01:41:45,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:41:45,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708505147, max timestamp=1733708505174Num uniques 859; 2024-12-09T01:41:45,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708505147, max timestamp=1733708505174Num uniques 859; 2024-12-09T01:41:45,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:41:45,178 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:41:45,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:45,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708565204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:45,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:45,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708565205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:45,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:45,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:45,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:45,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708565211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:45,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,219 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:45,233 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=262, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/dfc75619368649b8a3f5dc617e5a2d0a 2024-12-09T01:41:45,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/ac3ec016320e4668b890171699189d2e is 50, key is test_row_0/B:col10/1733708502958/Put/seqid=0 2024-12-09T01:41:45,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742025_1201 (size=12251) 2024-12-09T01:41:45,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:45,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708565313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:45,316 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:45,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:45,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708565323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:45,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:45,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:45,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708565324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:45,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:45,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:45,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708565523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:45,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:45,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708565533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:45,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:45,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:45,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708565544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:45,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:45,651 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/ac3ec016320e4668b890171699189d2e 2024-12-09T01:41:45,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/f7a2fa0fcd5b4e82afe440793241495c is 50, key is test_row_0/C:col10/1733708502958/Put/seqid=0 2024-12-09T01:41:45,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742026_1202 (size=12251) 2024-12-09T01:41:45,687 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/f7a2fa0fcd5b4e82afe440793241495c 2024-12-09T01:41:45,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/dfc75619368649b8a3f5dc617e5a2d0a as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/dfc75619368649b8a3f5dc617e5a2d0a 2024-12-09T01:41:45,703 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/dfc75619368649b8a3f5dc617e5a2d0a, entries=150, sequenceid=262, filesize=30.5 K 2024-12-09T01:41:45,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/ac3ec016320e4668b890171699189d2e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/ac3ec016320e4668b890171699189d2e 2024-12-09T01:41:45,711 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/ac3ec016320e4668b890171699189d2e, entries=150, sequenceid=262, filesize=12.0 K 2024-12-09T01:41:45,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/f7a2fa0fcd5b4e82afe440793241495c as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/f7a2fa0fcd5b4e82afe440793241495c 2024-12-09T01:41:45,718 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/f7a2fa0fcd5b4e82afe440793241495c, entries=150, sequenceid=262, filesize=12.0 K 2024-12-09T01:41:45,720 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for ca70d838c08a81d31de52aceb8d05bfd in 1355ms, sequenceid=262, compaction requested=false 2024-12-09T01:41:45,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:45,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:45,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-12-09T01:41:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-12-09T01:41:45,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-12-09T01:41:45,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1250 sec 2024-12-09T01:41:45,728 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.1330 sec 2024-12-09T01:41:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-09T01:41:45,744 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:41:45,745 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:41:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-12-09T01:41:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T01:41:45,747 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:41:45,748 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:41:45,749 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:41:45,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:45,841 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca70d838c08a81d31de52aceb8d05bfd 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-12-09T01:41:45,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=A 2024-12-09T01:41:45,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:45,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=B 2024-12-09T01:41:45,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:45,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=C 2024-12-09T01:41:45,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:45,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098cd1031959814fdb8059c390182a732f_ca70d838c08a81d31de52aceb8d05bfd is 50, key is test_row_0/A:col10/1733708505202/Put/seqid=0 2024-12-09T01:41:45,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T01:41:45,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708565858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:45,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708565859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:45,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:45,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:45,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742027_1203 (size=14994) 2024-12-09T01:41:45,863 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:45,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708565862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:45,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:45,869 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098cd1031959814fdb8059c390182a732f_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098cd1031959814fdb8059c390182a732f_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:45,870 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/02757be7af2e4798b052c62047faede5, store: [table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:45,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/02757be7af2e4798b052c62047faede5 is 175, key is test_row_0/A:col10/1733708505202/Put/seqid=0 2024-12-09T01:41:45,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742028_1204 (size=39949) 2024-12-09T01:41:45,884 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/02757be7af2e4798b052c62047faede5 2024-12-09T01:41:45,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/b3572547ce5e483e8f2e10d55b195254 is 50, key is test_row_0/B:col10/1733708505202/Put/seqid=0 2024-12-09T01:41:45,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-09T01:41:45,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:45,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:45,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:45,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:45,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:45,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:45,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742029_1205 (size=12301) 2024-12-09T01:41:45,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/b3572547ce5e483e8f2e10d55b195254 2024-12-09T01:41:45,928 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/8f05ad379b1f4b23a452804c3457faa7 is 50, key is test_row_0/C:col10/1733708505202/Put/seqid=0 2024-12-09T01:41:45,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742030_1206 (size=12301) 2024-12-09T01:41:45,943 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/8f05ad379b1f4b23a452804c3457faa7 2024-12-09T01:41:45,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/02757be7af2e4798b052c62047faede5 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/02757be7af2e4798b052c62047faede5 2024-12-09T01:41:45,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/02757be7af2e4798b052c62047faede5, entries=200, sequenceid=300, filesize=39.0 K 2024-12-09T01:41:45,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/b3572547ce5e483e8f2e10d55b195254 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/b3572547ce5e483e8f2e10d55b195254 2024-12-09T01:41:45,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/b3572547ce5e483e8f2e10d55b195254, entries=150, sequenceid=300, filesize=12.0 K 2024-12-09T01:41:45,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:45,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708565964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:45,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:45,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:45,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/8f05ad379b1f4b23a452804c3457faa7 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/8f05ad379b1f4b23a452804c3457faa7 2024-12-09T01:41:45,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/8f05ad379b1f4b23a452804c3457faa7, entries=150, sequenceid=300, filesize=12.0 K 2024-12-09T01:41:45,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for ca70d838c08a81d31de52aceb8d05bfd in 135ms, sequenceid=300, compaction requested=true 2024-12-09T01:41:45,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:45,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca70d838c08a81d31de52aceb8d05bfd:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:41:45,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:45,977 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:45,977 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:45,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca70d838c08a81d31de52aceb8d05bfd:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:41:45,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:45,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca70d838c08a81d31de52aceb8d05bfd:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:41:45,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:45,978 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37079 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:45,978 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): ca70d838c08a81d31de52aceb8d05bfd/B is initiating minor compaction (all files) 2024-12-09T01:41:45,978 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca70d838c08a81d31de52aceb8d05bfd/B in TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:45,979 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/7987c2d0270644639ea9af37de2a07d9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/ac3ec016320e4668b890171699189d2e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/b3572547ce5e483e8f2e10d55b195254] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp, totalSize=36.2 K 2024-12-09T01:41:45,979 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102635 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:45,979 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): ca70d838c08a81d31de52aceb8d05bfd/A is initiating minor compaction (all files) 2024-12-09T01:41:45,979 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca70d838c08a81d31de52aceb8d05bfd/A in TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:45,979 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/bcde21feb8304a2a849edc29a77caa78, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/dfc75619368649b8a3f5dc617e5a2d0a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/02757be7af2e4798b052c62047faede5] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp, totalSize=100.2 K 2024-12-09T01:41:45,979 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:45,979 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. files: [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/bcde21feb8304a2a849edc29a77caa78, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/dfc75619368649b8a3f5dc617e5a2d0a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/02757be7af2e4798b052c62047faede5] 2024-12-09T01:41:45,980 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 7987c2d0270644639ea9af37de2a07d9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733708501788 2024-12-09T01:41:45,980 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting bcde21feb8304a2a849edc29a77caa78, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733708501788 2024-12-09T01:41:45,981 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting ac3ec016320e4668b890171699189d2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733708502949 2024-12-09T01:41:45,981 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfc75619368649b8a3f5dc617e5a2d0a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733708502949 2024-12-09T01:41:45,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:45,983 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting b3572547ce5e483e8f2e10d55b195254, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733708505186 2024-12-09T01:41:45,983 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02757be7af2e4798b052c62047faede5, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733708505186 2024-12-09T01:41:45,983 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca70d838c08a81d31de52aceb8d05bfd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:41:45,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=A 2024-12-09T01:41:45,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:45,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=B 2024-12-09T01:41:45,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:45,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=C 2024-12-09T01:41:45,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:46,001 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:46,002 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca70d838c08a81d31de52aceb8d05bfd#B#compaction#177 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:46,003 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/2aabcabb0c404157ad542b10539ebc38 is 50, key is test_row_0/B:col10/1733708505202/Put/seqid=0 2024-12-09T01:41:46,016 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209662ec46a25d34349a57ed8dbf9a2f12c_ca70d838c08a81d31de52aceb8d05bfd store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:46,018 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209662ec46a25d34349a57ed8dbf9a2f12c_ca70d838c08a81d31de52aceb8d05bfd, store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:46,018 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209662ec46a25d34349a57ed8dbf9a2f12c_ca70d838c08a81d31de52aceb8d05bfd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:46,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d291d43f26454961b80663e0422650c6_ca70d838c08a81d31de52aceb8d05bfd is 50, key is test_row_0/A:col10/1733708505980/Put/seqid=0 2024-12-09T01:41:46,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742032_1208 (size=4469) 2024-12-09T01:41:46,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742033_1209 (size=12454) 2024-12-09T01:41:46,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742031_1207 (size=12779) 2024-12-09T01:41:46,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708506035, max timestamp=1733708506052Num uniques 859; 2024-12-09T01:41:46,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:41:46,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708506035, max timestamp=1733708506052Num uniques 859; 2024-12-09T01:41:46,055 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:41:46,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-09T01:41:46,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708506035, max timestamp=1733708506052Num uniques 859; 2024-12-09T01:41:46,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:41:46,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:46,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:46,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:46,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:46,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:46,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T01:41:46,115 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:46,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708566106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:46,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:46,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:46,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708566114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:46,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:46,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708566174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:46,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,181 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:46,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-09T01:41:46,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:46,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:46,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:46,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:46,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:46,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:46,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:46,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708566223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:46,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:46,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:46,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708566224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:46,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:46,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-09T01:41:46,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:46,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:46,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:46,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:46,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:46,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T01:41:46,437 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:46,437 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca70d838c08a81d31de52aceb8d05bfd#A#compaction#178 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:46,438 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/2910f7498c304b51aeae7b68ba606e04 is 175, key is test_row_0/A:col10/1733708505202/Put/seqid=0 2024-12-09T01:41:46,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:46,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708566434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:46,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:46,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708566434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:46,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:46,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,441 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:46,443 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d291d43f26454961b80663e0422650c6_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d291d43f26454961b80663e0422650c6_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:46,444 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/b95b16ffc35c438e9a3fd2b31feb3498, store: [table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:46,445 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/b95b16ffc35c438e9a3fd2b31feb3498 is 175, key is test_row_0/A:col10/1733708505980/Put/seqid=0 2024-12-09T01:41:46,453 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/2aabcabb0c404157ad542b10539ebc38 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/2aabcabb0c404157ad542b10539ebc38 2024-12-09T01:41:46,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742034_1210 (size=31733) 2024-12-09T01:41:46,461 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca70d838c08a81d31de52aceb8d05bfd/B of ca70d838c08a81d31de52aceb8d05bfd into 2aabcabb0c404157ad542b10539ebc38(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:46,461 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:46,461 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., storeName=ca70d838c08a81d31de52aceb8d05bfd/B, priority=13, startTime=1733708505977; duration=0sec 2024-12-09T01:41:46,462 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:46,462 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca70d838c08a81d31de52aceb8d05bfd:B 2024-12-09T01:41:46,462 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:46,462 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/2910f7498c304b51aeae7b68ba606e04 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/2910f7498c304b51aeae7b68ba606e04 2024-12-09T01:41:46,463 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37079 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:46,463 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): ca70d838c08a81d31de52aceb8d05bfd/C is initiating minor compaction (all files) 2024-12-09T01:41:46,463 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca70d838c08a81d31de52aceb8d05bfd/C in TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:46,464 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/0baaa9f8e0df44fa87619e1274b3901e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/f7a2fa0fcd5b4e82afe440793241495c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/8f05ad379b1f4b23a452804c3457faa7] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp, totalSize=36.2 K 2024-12-09T01:41:46,465 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 0baaa9f8e0df44fa87619e1274b3901e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1733708501788 2024-12-09T01:41:46,465 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting f7a2fa0fcd5b4e82afe440793241495c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1733708502949 2024-12-09T01:41:46,466 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f05ad379b1f4b23a452804c3457faa7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733708505186 2024-12-09T01:41:46,470 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca70d838c08a81d31de52aceb8d05bfd/A of ca70d838c08a81d31de52aceb8d05bfd into 2910f7498c304b51aeae7b68ba606e04(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:46,470 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:46,470 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., storeName=ca70d838c08a81d31de52aceb8d05bfd/A, priority=13, startTime=1733708505977; duration=0sec 2024-12-09T01:41:46,470 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:46,470 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca70d838c08a81d31de52aceb8d05bfd:A 2024-12-09T01:41:46,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742035_1211 (size=31255) 2024-12-09T01:41:46,479 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=310, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/b95b16ffc35c438e9a3fd2b31feb3498 2024-12-09T01:41:46,487 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca70d838c08a81d31de52aceb8d05bfd#C#compaction#180 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:46,488 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/d971de3c5b13429b8128b443ee9c488e is 50, key is test_row_0/C:col10/1733708505202/Put/seqid=0 2024-12-09T01:41:46,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708566485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:46,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:46,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/4d1fcbb4bc8743ae82d071e9020763fe is 50, key is test_row_0/B:col10/1733708505980/Put/seqid=0 2024-12-09T01:41:46,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742036_1212 (size=12779) 2024-12-09T01:41:46,511 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/d971de3c5b13429b8128b443ee9c488e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/d971de3c5b13429b8128b443ee9c488e 2024-12-09T01:41:46,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742037_1213 (size=12301) 2024-12-09T01:41:46,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/4d1fcbb4bc8743ae82d071e9020763fe 2024-12-09T01:41:46,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-09T01:41:46,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:46,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:46,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:46,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:46,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:46,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:46,527 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca70d838c08a81d31de52aceb8d05bfd/C of ca70d838c08a81d31de52aceb8d05bfd into d971de3c5b13429b8128b443ee9c488e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:46,527 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:46,527 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., storeName=ca70d838c08a81d31de52aceb8d05bfd/C, priority=13, startTime=1733708505977; duration=0sec 2024-12-09T01:41:46,527 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:46,527 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca70d838c08a81d31de52aceb8d05bfd:C 2024-12-09T01:41:46,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/d8cb9817a0a4475cb95aaa651e2c7901 is 50, key is test_row_0/C:col10/1733708505980/Put/seqid=0 2024-12-09T01:41:46,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742038_1214 (size=12301) 2024-12-09T01:41:46,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/d8cb9817a0a4475cb95aaa651e2c7901 2024-12-09T01:41:46,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/b95b16ffc35c438e9a3fd2b31feb3498 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/b95b16ffc35c438e9a3fd2b31feb3498 2024-12-09T01:41:46,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/b95b16ffc35c438e9a3fd2b31feb3498, entries=150, sequenceid=310, filesize=30.5 K 2024-12-09T01:41:46,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/4d1fcbb4bc8743ae82d071e9020763fe as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/4d1fcbb4bc8743ae82d071e9020763fe 2024-12-09T01:41:46,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/4d1fcbb4bc8743ae82d071e9020763fe, entries=150, sequenceid=310, filesize=12.0 K 2024-12-09T01:41:46,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/d8cb9817a0a4475cb95aaa651e2c7901 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/d8cb9817a0a4475cb95aaa651e2c7901 2024-12-09T01:41:46,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/d8cb9817a0a4475cb95aaa651e2c7901, entries=150, sequenceid=310, filesize=12.0 K 2024-12-09T01:41:46,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ca70d838c08a81d31de52aceb8d05bfd in 583ms, sequenceid=310, compaction requested=false 2024-12-09T01:41:46,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:46,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:46,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca70d838c08a81d31de52aceb8d05bfd 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-09T01:41:46,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=A 2024-12-09T01:41:46,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:46,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=B 2024-12-09T01:41:46,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:46,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=C 2024-12-09T01:41:46,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:46,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b9eb87243bce42848d328656bab97f26_ca70d838c08a81d31de52aceb8d05bfd is 50, key is test_row_0/A:col10/1733708506099/Put/seqid=0 2024-12-09T01:41:46,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-09T01:41:46,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:46,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:46,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:46,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:46,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:46,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742039_1215 (size=14994) 2024-12-09T01:41:46,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:46,675 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:46,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708566675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:46,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:46,681 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b9eb87243bce42848d328656bab97f26_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b9eb87243bce42848d328656bab97f26_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:46,682 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/430549c873a649fbace40647a3fee393, store: [table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:46,683 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/430549c873a649fbace40647a3fee393 is 175, key is test_row_0/A:col10/1733708506099/Put/seqid=0 2024-12-09T01:41:46,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:46,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708566677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:46,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:46,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742040_1216 (size=39949) 2024-12-09T01:41:46,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:46,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708566743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:46,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:46,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708566753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,757 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:46,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:46,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708566783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:46,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:46,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:46,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708566793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:46,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-09T01:41:46,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:46,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:46,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:46,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:46,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:46,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:46,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T01:41:46,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-09T01:41:46,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:46,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:46,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:46,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:46,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:46,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:46,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:46,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708566993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:46,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:46,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:47,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:47,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708567003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:47,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:47,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:47,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:47,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:47,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708567004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:47,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:47,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:47,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:47,088 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=350, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/430549c873a649fbace40647a3fee393 2024-12-09T01:41:47,098 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/35b21c2ae8dd47fd8c05c98b7f7c369f is 50, key is test_row_0/B:col10/1733708506099/Put/seqid=0 2024-12-09T01:41:47,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742041_1217 (size=12301) 2024-12-09T01:41:47,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/35b21c2ae8dd47fd8c05c98b7f7c369f 2024-12-09T01:41:47,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/b298c2e6b13343d7881ba15ec47d5d60 is 50, key is test_row_0/C:col10/1733708506099/Put/seqid=0 2024-12-09T01:41:47,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742042_1218 (size=12301) 2024-12-09T01:41:47,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-09T01:41:47,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:47,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:47,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:47,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:47,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:47,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:47,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:47,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708567254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:47,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:47,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:47,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:47,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:47,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708567264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:47,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:47,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-09T01:41:47,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:47,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:47,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:47,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:47,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:47,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:47,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708567303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:47,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:47,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:47,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:47,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:47,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708567313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:47,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:47,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:47,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:47,441 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-09T01:41:47,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:47,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:47,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:47,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:47,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:47,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:47,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/b298c2e6b13343d7881ba15ec47d5d60 2024-12-09T01:41:47,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/430549c873a649fbace40647a3fee393 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/430549c873a649fbace40647a3fee393 2024-12-09T01:41:47,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/430549c873a649fbace40647a3fee393, entries=200, sequenceid=350, filesize=39.0 K 2024-12-09T01:41:47,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/35b21c2ae8dd47fd8c05c98b7f7c369f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/35b21c2ae8dd47fd8c05c98b7f7c369f 2024-12-09T01:41:47,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/35b21c2ae8dd47fd8c05c98b7f7c369f, entries=150, sequenceid=350, filesize=12.0 K 2024-12-09T01:41:47,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/b298c2e6b13343d7881ba15ec47d5d60 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/b298c2e6b13343d7881ba15ec47d5d60 2024-12-09T01:41:47,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/b298c2e6b13343d7881ba15ec47d5d60, entries=150, sequenceid=350, filesize=12.0 K 2024-12-09T01:41:47,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for ca70d838c08a81d31de52aceb8d05bfd in 885ms, sequenceid=350, compaction requested=true 2024-12-09T01:41:47,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:47,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca70d838c08a81d31de52aceb8d05bfd:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:41:47,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:47,541 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:47,541 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:47,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca70d838c08a81d31de52aceb8d05bfd:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:41:47,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:47,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca70d838c08a81d31de52aceb8d05bfd:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:41:47,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:47,542 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102937 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:47,542 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): ca70d838c08a81d31de52aceb8d05bfd/A is initiating minor compaction (all files) 2024-12-09T01:41:47,542 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:47,542 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca70d838c08a81d31de52aceb8d05bfd/A in TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:47,542 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): ca70d838c08a81d31de52aceb8d05bfd/B is initiating minor compaction (all files) 2024-12-09T01:41:47,542 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/2910f7498c304b51aeae7b68ba606e04, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/b95b16ffc35c438e9a3fd2b31feb3498, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/430549c873a649fbace40647a3fee393] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp, totalSize=100.5 K 2024-12-09T01:41:47,542 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca70d838c08a81d31de52aceb8d05bfd/B in TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:47,542 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:47,542 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. files: [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/2910f7498c304b51aeae7b68ba606e04, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/b95b16ffc35c438e9a3fd2b31feb3498, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/430549c873a649fbace40647a3fee393] 2024-12-09T01:41:47,542 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/2aabcabb0c404157ad542b10539ebc38, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/4d1fcbb4bc8743ae82d071e9020763fe, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/35b21c2ae8dd47fd8c05c98b7f7c369f] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp, totalSize=36.5 K 2024-12-09T01:41:47,544 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2910f7498c304b51aeae7b68ba606e04, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733708505186 2024-12-09T01:41:47,544 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aabcabb0c404157ad542b10539ebc38, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733708505186 2024-12-09T01:41:47,544 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d1fcbb4bc8743ae82d071e9020763fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733708505845 2024-12-09T01:41:47,544 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting b95b16ffc35c438e9a3fd2b31feb3498, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733708505845 2024-12-09T01:41:47,545 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 430549c873a649fbace40647a3fee393, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733708506076 2024-12-09T01:41:47,545 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 35b21c2ae8dd47fd8c05c98b7f7c369f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733708506076 2024-12-09T01:41:47,552 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:47,553 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca70d838c08a81d31de52aceb8d05bfd#B#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:47,553 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/7fbfdc7d6e134b6aa2dddb430fac0f31 is 50, key is test_row_0/B:col10/1733708506099/Put/seqid=0 2024-12-09T01:41:47,554 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209d2677c4e53784b569ac13c21ac5d323d_ca70d838c08a81d31de52aceb8d05bfd store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:47,556 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209d2677c4e53784b569ac13c21ac5d323d_ca70d838c08a81d31de52aceb8d05bfd, store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:47,556 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d2677c4e53784b569ac13c21ac5d323d_ca70d838c08a81d31de52aceb8d05bfd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:47,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742044_1220 (size=4469) 2024-12-09T01:41:47,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742043_1219 (size=12881) 2024-12-09T01:41:47,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-12-09T01:41:47,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:47,594 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing ca70d838c08a81d31de52aceb8d05bfd 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-09T01:41:47,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=A 2024-12-09T01:41:47,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:47,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=B 2024-12-09T01:41:47,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:47,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=C 2024-12-09T01:41:47,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:47,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120935aca110df764a39b22ff8968d25cd4e_ca70d838c08a81d31de52aceb8d05bfd is 50, key is test_row_0/A:col10/1733708506668/Put/seqid=0 2024-12-09T01:41:47,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742045_1221 (size=12454) 2024-12-09T01:41:47,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:47,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:47,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708507841, max timestamp=1733708507858Num uniques 859; 2024-12-09T01:41:47,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:41:47,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708507841, max timestamp=1733708507858Num uniques 859; 2024-12-09T01:41:47,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708507841, max timestamp=1733708507858Num uniques 859; 2024-12-09T01:41:47,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:41:47,862 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:41:47,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T01:41:47,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:47,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708567921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:47,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:47,924 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:47,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708567922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:47,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:47,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:47,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:47,965 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca70d838c08a81d31de52aceb8d05bfd#A#compaction#187 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:47,966 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/ca9d8872599a420aabd3dbc8cf6202d3 is 175, key is test_row_0/A:col10/1733708506099/Put/seqid=0 2024-12-09T01:41:47,971 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/7fbfdc7d6e134b6aa2dddb430fac0f31 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/7fbfdc7d6e134b6aa2dddb430fac0f31 2024-12-09T01:41:47,976 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca70d838c08a81d31de52aceb8d05bfd/B of ca70d838c08a81d31de52aceb8d05bfd into 7fbfdc7d6e134b6aa2dddb430fac0f31(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:47,976 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:47,976 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., storeName=ca70d838c08a81d31de52aceb8d05bfd/B, priority=13, startTime=1733708507541; duration=0sec 2024-12-09T01:41:47,977 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:47,977 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca70d838c08a81d31de52aceb8d05bfd:B 2024-12-09T01:41:47,977 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:47,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742046_1222 (size=31835) 2024-12-09T01:41:47,978 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:47,978 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): ca70d838c08a81d31de52aceb8d05bfd/C is initiating minor compaction (all files) 2024-12-09T01:41:47,978 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca70d838c08a81d31de52aceb8d05bfd/C in TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:47,978 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/d971de3c5b13429b8128b443ee9c488e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/d8cb9817a0a4475cb95aaa651e2c7901, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/b298c2e6b13343d7881ba15ec47d5d60] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp, totalSize=36.5 K 2024-12-09T01:41:47,979 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting d971de3c5b13429b8128b443ee9c488e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1733708505186 2024-12-09T01:41:47,980 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting d8cb9817a0a4475cb95aaa651e2c7901, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1733708505845 2024-12-09T01:41:47,980 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting b298c2e6b13343d7881ba15ec47d5d60, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733708506076 2024-12-09T01:41:47,984 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/ca9d8872599a420aabd3dbc8cf6202d3 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/ca9d8872599a420aabd3dbc8cf6202d3 2024-12-09T01:41:47,991 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca70d838c08a81d31de52aceb8d05bfd/A of ca70d838c08a81d31de52aceb8d05bfd into ca9d8872599a420aabd3dbc8cf6202d3(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:47,991 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:47,991 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., storeName=ca70d838c08a81d31de52aceb8d05bfd/A, priority=13, startTime=1733708507541; duration=0sec 2024-12-09T01:41:47,991 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:47,991 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca70d838c08a81d31de52aceb8d05bfd:A 2024-12-09T01:41:47,995 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca70d838c08a81d31de52aceb8d05bfd#C#compaction#189 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:47,995 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/6fba8204ee92413886a1e20a3fb8b317 is 50, key is test_row_0/C:col10/1733708506099/Put/seqid=0 2024-12-09T01:41:48,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742047_1223 (size=12881) 2024-12-09T01:41:48,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:48,010 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/6fba8204ee92413886a1e20a3fb8b317 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/6fba8204ee92413886a1e20a3fb8b317 2024-12-09T01:41:48,013 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120935aca110df764a39b22ff8968d25cd4e_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120935aca110df764a39b22ff8968d25cd4e_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:48,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/4f6703a0d54b4726bf693a9de6a9854c, store: [table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:48,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/4f6703a0d54b4726bf693a9de6a9854c is 175, key is test_row_0/A:col10/1733708506668/Put/seqid=0 2024-12-09T01:41:48,018 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca70d838c08a81d31de52aceb8d05bfd/C of ca70d838c08a81d31de52aceb8d05bfd into 6fba8204ee92413886a1e20a3fb8b317(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:48,018 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:48,018 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., storeName=ca70d838c08a81d31de52aceb8d05bfd/C, priority=13, startTime=1733708507541; duration=0sec 2024-12-09T01:41:48,018 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:48,018 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca70d838c08a81d31de52aceb8d05bfd:C 2024-12-09T01:41:48,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742048_1224 (size=31255) 2024-12-09T01:41:48,025 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=359, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/4f6703a0d54b4726bf693a9de6a9854c 2024-12-09T01:41:48,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:48,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708568023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:48,028 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_1' in ca70d838c08a81d31de52aceb8d05bfd of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:41:48,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:48,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:48,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708568034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:48,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:48,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708568034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:48,039 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:48,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/6fb92d81cb554132aa5e805f3986aad3 is 50, key is test_row_0/B:col10/1733708506668/Put/seqid=0 2024-12-09T01:41:48,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742049_1225 (size=12301) 2024-12-09T01:41:48,052 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/6fb92d81cb554132aa5e805f3986aad3 2024-12-09T01:41:48,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/874d64dbf6ae4962affc9552f6fe3099 is 50, key is test_row_0/C:col10/1733708506668/Put/seqid=0 2024-12-09T01:41:48,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742050_1226 (size=12301) 2024-12-09T01:41:48,076 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/874d64dbf6ae4962affc9552f6fe3099 2024-12-09T01:41:48,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/4f6703a0d54b4726bf693a9de6a9854c as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/4f6703a0d54b4726bf693a9de6a9854c 2024-12-09T01:41:48,095 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/4f6703a0d54b4726bf693a9de6a9854c, entries=150, sequenceid=359, filesize=30.5 K 2024-12-09T01:41:48,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/6fb92d81cb554132aa5e805f3986aad3 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/6fb92d81cb554132aa5e805f3986aad3 2024-12-09T01:41:48,101 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/6fb92d81cb554132aa5e805f3986aad3, entries=150, sequenceid=359, filesize=12.0 K 2024-12-09T01:41:48,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/874d64dbf6ae4962affc9552f6fe3099 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/874d64dbf6ae4962affc9552f6fe3099 2024-12-09T01:41:48,106 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/874d64dbf6ae4962affc9552f6fe3099, entries=150, sequenceid=359, filesize=12.0 K 2024-12-09T01:41:48,108 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=147.60 KB/151140 for ca70d838c08a81d31de52aceb8d05bfd in 514ms, sequenceid=359, compaction requested=false 2024-12-09T01:41:48,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:48,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:48,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-12-09T01:41:48,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-12-09T01:41:48,111 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-12-09T01:41:48,111 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3600 sec 2024-12-09T01:41:48,113 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.3670 sec 2024-12-09T01:41:48,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733708507899, max timestamp=1733708508246Num uniques 1118; 2024-12-09T01:41:48,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-09T01:41:48,252 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca70d838c08a81d31de52aceb8d05bfd 3/3 column families, dataSize=111.82 KB heapSize=312.52 KB 2024-12-09T01:41:48,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:48,256 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733708507899, max timestamp=1733708508246Num uniques 1118; 2024-12-09T01:41:48,256 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-09T01:41:48,256 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733708507899, max timestamp=1733708508246Num uniques 1118; 2024-12-09T01:41:48,256 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=7688, suffix heap size=144240,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=168 2024-12-09T01:41:48,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=A 2024-12-09T01:41:48,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:48,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=B 2024-12-09T01:41:48,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:48,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=C 2024-12-09T01:41:48,257 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:48,272 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099b0ee88ff5c54d0ebf26d4276be3c8e4_ca70d838c08a81d31de52aceb8d05bfd is 50, key is test_row_0/A:col10/1733708508251/Put/seqid=0 2024-12-09T01:41:48,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708508273, max timestamp=1733708508304Num uniques 859; 2024-12-09T01:41:48,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:41:48,315 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708508273, max timestamp=1733708508304Num uniques 859; 2024-12-09T01:41:48,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:41:48,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708508273, max timestamp=1733708508304Num uniques 859; 2024-12-09T01:41:48,317 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:41:48,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742051_1227 (size=17534) 2024-12-09T01:41:48,375 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733708508332, max timestamp=1733708508370Num uniques 1068; 2024-12-09T01:41:48,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-12-09T01:41:48,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733708508332, max timestamp=1733708508370Num uniques 1068; 2024-12-09T01:41:48,376 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-12-09T01:41:48,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1733708508332, max timestamp=1733708508370Num uniques 1068; 2024-12-09T01:41:48,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-12-09T01:41:48,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:48,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708568408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:48,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:48,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:48,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708568409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708568410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:48,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708568412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:48,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:48,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708568523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:48,529 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:48,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:48,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708568533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:48,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:48,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708568533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:48,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:48,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:48,542 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708568533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:48,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:48,728 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:48,733 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099b0ee88ff5c54d0ebf26d4276be3c8e4_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099b0ee88ff5c54d0ebf26d4276be3c8e4_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:48,734 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/ce041a9c34484d448d056aa8fed32eb9, store: [table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:48,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/ce041a9c34484d448d056aa8fed32eb9 is 175, key is test_row_0/A:col10/1733708508251/Put/seqid=0 2024-12-09T01:41:48,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:48,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708568733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:48,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742052_1228 (size=48639) 2024-12-09T01:41:48,757 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:48,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708568753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:48,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:48,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708568754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:48,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708568754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:48,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:48,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:48,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:49,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:49,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708569043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:49,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,048 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:49,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:49,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708569064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:49,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:49,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708569073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:49,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708569073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:49,140 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=401, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/ce041a9c34484d448d056aa8fed32eb9 2024-12-09T01:41:49,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/5f6026d1447d4f1bb0d8b78595ed9962 is 50, key is test_row_0/B:col10/1733708508251/Put/seqid=0 2024-12-09T01:41:49,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742053_1229 (size=12301) 2024-12-09T01:41:49,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/5f6026d1447d4f1bb0d8b78595ed9962 2024-12-09T01:41:49,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/0b7df81e659f4d70b83b6c595440000e is 50, key is test_row_0/C:col10/1733708508251/Put/seqid=0 2024-12-09T01:41:49,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742054_1230 (size=12301) 2024-12-09T01:41:49,173 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/0b7df81e659f4d70b83b6c595440000e 2024-12-09T01:41:49,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/ce041a9c34484d448d056aa8fed32eb9 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/ce041a9c34484d448d056aa8fed32eb9 2024-12-09T01:41:49,182 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/ce041a9c34484d448d056aa8fed32eb9, entries=250, sequenceid=401, filesize=47.5 K 2024-12-09T01:41:49,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/5f6026d1447d4f1bb0d8b78595ed9962 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/5f6026d1447d4f1bb0d8b78595ed9962 2024-12-09T01:41:49,188 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/5f6026d1447d4f1bb0d8b78595ed9962, entries=150, sequenceid=401, filesize=12.0 K 2024-12-09T01:41:49,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/0b7df81e659f4d70b83b6c595440000e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/0b7df81e659f4d70b83b6c595440000e 2024-12-09T01:41:49,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/0b7df81e659f4d70b83b6c595440000e, entries=150, sequenceid=401, filesize=12.0 K 2024-12-09T01:41:49,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=154.31 KB/158010 for ca70d838c08a81d31de52aceb8d05bfd in 944ms, sequenceid=401, compaction requested=true 2024-12-09T01:41:49,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:49,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca70d838c08a81d31de52aceb8d05bfd:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:41:49,196 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:49,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca70d838c08a81d31de52aceb8d05bfd:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:41:49,197 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:49,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:49,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca70d838c08a81d31de52aceb8d05bfd:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:41:49,197 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:49,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:49,198 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111729 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:49,198 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): ca70d838c08a81d31de52aceb8d05bfd/A is initiating minor compaction (all files) 2024-12-09T01:41:49,198 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca70d838c08a81d31de52aceb8d05bfd/A in TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:49,198 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/ca9d8872599a420aabd3dbc8cf6202d3, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/4f6703a0d54b4726bf693a9de6a9854c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/ce041a9c34484d448d056aa8fed32eb9] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp, totalSize=109.1 K 2024-12-09T01:41:49,198 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:49,198 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. files: [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/ca9d8872599a420aabd3dbc8cf6202d3, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/4f6703a0d54b4726bf693a9de6a9854c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/ce041a9c34484d448d056aa8fed32eb9] 2024-12-09T01:41:49,199 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:49,199 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca9d8872599a420aabd3dbc8cf6202d3, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733708506076 2024-12-09T01:41:49,199 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): ca70d838c08a81d31de52aceb8d05bfd/B is initiating minor compaction (all files) 2024-12-09T01:41:49,199 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca70d838c08a81d31de52aceb8d05bfd/B in TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:49,199 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/7fbfdc7d6e134b6aa2dddb430fac0f31, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/6fb92d81cb554132aa5e805f3986aad3, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/5f6026d1447d4f1bb0d8b78595ed9962] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp, totalSize=36.6 K 2024-12-09T01:41:49,199 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f6703a0d54b4726bf693a9de6a9854c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733708506668 2024-12-09T01:41:49,200 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fbfdc7d6e134b6aa2dddb430fac0f31, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733708506076 2024-12-09T01:41:49,200 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce041a9c34484d448d056aa8fed32eb9, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733708507899 2024-12-09T01:41:49,200 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fb92d81cb554132aa5e805f3986aad3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733708506668 2024-12-09T01:41:49,201 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f6026d1447d4f1bb0d8b78595ed9962, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733708507909 2024-12-09T01:41:49,212 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:49,221 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca70d838c08a81d31de52aceb8d05bfd#B#compaction#196 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:49,222 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/d565b778838142ab8a07c3c64d3d04fd is 50, key is test_row_0/B:col10/1733708508251/Put/seqid=0 2024-12-09T01:41:49,229 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209f2cdc8f194354163a57fa7fd3322ec0d_ca70d838c08a81d31de52aceb8d05bfd store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:49,232 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209f2cdc8f194354163a57fa7fd3322ec0d_ca70d838c08a81d31de52aceb8d05bfd, store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:49,232 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f2cdc8f194354163a57fa7fd3322ec0d_ca70d838c08a81d31de52aceb8d05bfd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:49,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742055_1231 (size=12983) 2024-12-09T01:41:49,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742056_1232 (size=4469) 2024-12-09T01:41:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:49,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca70d838c08a81d31de52aceb8d05bfd 3/3 column families, dataSize=161.02 KB heapSize=450.47 KB 2024-12-09T01:41:49,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=A 2024-12-09T01:41:49,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T01:41:49,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=B 2024-12-09T01:41:49,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T01:41:49,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=C 2024-12-09T01:41:49,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T01:41:49,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097e193e11367c429bbc365eeb08da5c98_ca70d838c08a81d31de52aceb8d05bfd is 50, key is test_row_0/A:col10/1733708509565/Put/seqid=0 2024-12-09T01:41:49,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742057_1233 (size=14994) 2024-12-09T01:41:49,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708569585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:49,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:49,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708569585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:49,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:49,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708569585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:49,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:49,590 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708569586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:49,668 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/d565b778838142ab8a07c3c64d3d04fd as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/d565b778838142ab8a07c3c64d3d04fd 2024-12-09T01:41:49,673 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca70d838c08a81d31de52aceb8d05bfd/B of ca70d838c08a81d31de52aceb8d05bfd into d565b778838142ab8a07c3c64d3d04fd(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:49,673 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:49,673 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., storeName=ca70d838c08a81d31de52aceb8d05bfd/B, priority=13, startTime=1733708509196; duration=0sec 2024-12-09T01:41:49,673 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:49,673 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca70d838c08a81d31de52aceb8d05bfd:B 2024-12-09T01:41:49,673 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:49,675 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:49,675 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): ca70d838c08a81d31de52aceb8d05bfd/C is initiating minor compaction (all files) 2024-12-09T01:41:49,675 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca70d838c08a81d31de52aceb8d05bfd/C in TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:49,675 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/6fba8204ee92413886a1e20a3fb8b317, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/874d64dbf6ae4962affc9552f6fe3099, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/0b7df81e659f4d70b83b6c595440000e] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp, totalSize=36.6 K 2024-12-09T01:41:49,675 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fba8204ee92413886a1e20a3fb8b317, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1733708506076 2024-12-09T01:41:49,676 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 874d64dbf6ae4962affc9552f6fe3099, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1733708506668 2024-12-09T01:41:49,676 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b7df81e659f4d70b83b6c595440000e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733708507909 2024-12-09T01:41:49,685 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca70d838c08a81d31de52aceb8d05bfd#C#compaction#198 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:49,686 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/87c76479d2254e109b83f287f43b1603 is 50, key is test_row_0/C:col10/1733708508251/Put/seqid=0 2024-12-09T01:41:49,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742058_1234 (size=12983) 2024-12-09T01:41:49,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:49,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708569694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:49,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:49,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708569694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:49,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:49,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,699 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708569695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:49,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:49,704 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/87c76479d2254e109b83f287f43b1603 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/87c76479d2254e109b83f287f43b1603 2024-12-09T01:41:49,704 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca70d838c08a81d31de52aceb8d05bfd#A#compaction#195 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:49,705 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/dc6f8f9ea36e4c25bd1486bbaaf5e39b is 175, key is test_row_0/A:col10/1733708508251/Put/seqid=0 2024-12-09T01:41:49,718 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca70d838c08a81d31de52aceb8d05bfd/C of ca70d838c08a81d31de52aceb8d05bfd into 87c76479d2254e109b83f287f43b1603(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:49,718 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:49,718 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., storeName=ca70d838c08a81d31de52aceb8d05bfd/C, priority=13, startTime=1733708509197; duration=0sec 2024-12-09T01:41:49,718 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:49,718 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca70d838c08a81d31de52aceb8d05bfd:C 2024-12-09T01:41:49,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742059_1235 (size=31937) 2024-12-09T01:41:49,728 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/dc6f8f9ea36e4c25bd1486bbaaf5e39b as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/dc6f8f9ea36e4c25bd1486bbaaf5e39b 2024-12-09T01:41:49,737 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca70d838c08a81d31de52aceb8d05bfd/A of ca70d838c08a81d31de52aceb8d05bfd into dc6f8f9ea36e4c25bd1486bbaaf5e39b(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:49,737 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:49,737 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., storeName=ca70d838c08a81d31de52aceb8d05bfd/A, priority=13, startTime=1733708509196; duration=0sec 2024-12-09T01:41:49,737 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:49,738 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca70d838c08a81d31de52aceb8d05bfd:A 2024-12-09T01:41:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-09T01:41:49,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:49,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:49,914 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:41:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708569904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708569904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:49,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708569904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:49,916 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:41:49,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:49,917 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-12-09T01:41:49,918 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:41:49,919 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:41:49,920 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:41:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-09T01:41:49,983 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:49,990 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097e193e11367c429bbc365eeb08da5c98_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097e193e11367c429bbc365eeb08da5c98_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:49,992 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/5aed1fd9c5cd422e9596bc57795f283c, store: [table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:49,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/5aed1fd9c5cd422e9596bc57795f283c is 175, key is test_row_0/A:col10/1733708509565/Put/seqid=0 2024-12-09T01:41:50,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742060_1236 (size=39949) 2024-12-09T01:41:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-09T01:41:50,044 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:50,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60552 deadline: 1733708570043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:50,044 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_1' in ca70d838c08a81d31de52aceb8d05bfd of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:41:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:50,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-09T01:41:50,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:50,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:50,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:50,075 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:50,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:50,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:50,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:50,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708570223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:50,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:50,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708570223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:50,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:50,230 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-09T01:41:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:50,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:50,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:50,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:50,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:50,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:50,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708570224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:50,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,231 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:50,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:50,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-09T01:41:50,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-09T01:41:50,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:50,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:50,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:50,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:50,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:50,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:50,411 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=456, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/5aed1fd9c5cd422e9596bc57795f283c 2024-12-09T01:41:50,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/244915fb945a4cc99a4790a47a834393 is 50, key is test_row_0/B:col10/1733708509565/Put/seqid=0 2024-12-09T01:41:50,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742061_1237 (size=12301) 2024-12-09T01:41:50,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/244915fb945a4cc99a4790a47a834393 2024-12-09T01:41:50,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/6202c95b8ecc4754ad243435ca018840 is 50, key is test_row_0/C:col10/1733708509565/Put/seqid=0 2024-12-09T01:41:50,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742062_1238 (size=12301) 2024-12-09T01:41:50,535 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/6202c95b8ecc4754ad243435ca018840 2024-12-09T01:41:50,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-09T01:41:50,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:50,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:50,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:50,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:50,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:50,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:50,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-09T01:41:50,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/5aed1fd9c5cd422e9596bc57795f283c as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5aed1fd9c5cd422e9596bc57795f283c 2024-12-09T01:41:50,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5aed1fd9c5cd422e9596bc57795f283c, entries=200, sequenceid=456, filesize=39.0 K 2024-12-09T01:41:50,562 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/244915fb945a4cc99a4790a47a834393 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/244915fb945a4cc99a4790a47a834393 2024-12-09T01:41:50,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/244915fb945a4cc99a4790a47a834393, entries=150, sequenceid=456, filesize=12.0 K 2024-12-09T01:41:50,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/6202c95b8ecc4754ad243435ca018840 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/6202c95b8ecc4754ad243435ca018840 2024-12-09T01:41:50,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/6202c95b8ecc4754ad243435ca018840, entries=150, sequenceid=456, filesize=12.0 K 2024-12-09T01:41:50,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~450.42 KB/461232, currentSize=26.84 KB/27480 for ca70d838c08a81d31de52aceb8d05bfd in 1014ms, sequenceid=456, compaction requested=false 2024-12-09T01:41:50,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:50,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:50,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ca70d838c08a81d31de52aceb8d05bfd 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:41:50,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=A 2024-12-09T01:41:50,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:50,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=B 2024-12-09T01:41:50,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:50,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=C 2024-12-09T01:41:50,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:50,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:50,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091f26ceb4d5a944bba39629051fafce6c_ca70d838c08a81d31de52aceb8d05bfd is 50, key is test_row_0/A:col10/1733708510596/Put/seqid=0 2024-12-09T01:41:50,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-09T01:41:50,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:50,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:50,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:50,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:50,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:50,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:50,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742063_1239 (size=17534) 2024-12-09T01:41:50,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708510730, max timestamp=1733708510759Num uniques 859; 2024-12-09T01:41:50,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708510730, max timestamp=1733708510759Num uniques 859; 2024-12-09T01:41:50,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:41:50,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:41:50,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708510730, max timestamp=1733708510759Num uniques 859; 2024-12-09T01:41:50,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:41:50,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:50,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708570812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:50,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:50,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708570813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:50,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:50,835 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708570814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:50,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:50,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:50,844 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:50,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708570835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,845 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:50,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-09T01:41:50,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:50,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:50,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:50,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:50,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:50,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:50,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708570944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:50,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708570944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:50,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:50,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708570945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:50,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:50,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708570953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:50,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:50,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:51,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-09T01:41:51,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:51,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:51,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:51,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:51,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:51,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-09T01:41:51,125 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:51,132 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412091f26ceb4d5a944bba39629051fafce6c_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091f26ceb4d5a944bba39629051fafce6c_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:51,133 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/be3b222bf7754177ab4ad6555847e134, store: [table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:51,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/be3b222bf7754177ab4ad6555847e134 is 175, key is test_row_0/A:col10/1733708510596/Put/seqid=0 2024-12-09T01:41:51,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-09T01:41:51,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:51,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:51,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:51,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:51,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:51,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:51,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742064_1240 (size=48639) 2024-12-09T01:41:51,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708571163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:51,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:51,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:51,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:51,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708571164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:51,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:51,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708571164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:51,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708571175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:51,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-09T01:41:51,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:51,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:51,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:51,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:51,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:51,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:51,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-09T01:41:51,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:51,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:51,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:51,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:51,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:51,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:51,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708571483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:51,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708571483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:51,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:51,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:51,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:51,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708571483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:51,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:51,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:51,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:51,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:51,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:51,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708571483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:51,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:51,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:51,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:51,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:51,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:51,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:51,567 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=469, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/be3b222bf7754177ab4ad6555847e134 2024-12-09T01:41:51,596 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/2b731b66a42c4cd7bd8768ff017f71c1 is 50, key is test_row_0/B:col10/1733708510596/Put/seqid=0 2024-12-09T01:41:51,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-09T01:41:51,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:51,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:51,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:51,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:51,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:51,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742065_1241 (size=12301) 2024-12-09T01:41:51,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-09T01:41:51,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:51,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:51,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:51,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:51,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:51,928 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-09T01:41:51,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:51,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:51,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:51,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:51,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:51,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:51,959 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T01:41:52,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:52,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60566 deadline: 1733708572005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:52,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:52,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60536 deadline: 1733708572005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:52,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:52,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:52,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:52,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60582 deadline: 1733708572005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:52,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:52,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:52,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:52,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:41:52,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60528 deadline: 1733708572014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:52,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:52,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:41:52,020 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:41:52,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/2b731b66a42c4cd7bd8768ff017f71c1 2024-12-09T01:41:52,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/1dd7dd8e6a5d4030a82e74c85720ed26 is 50, key is test_row_0/C:col10/1733708510596/Put/seqid=0 2024-12-09T01:41:52,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-09T01:41:52,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-09T01:41:52,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:52,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:52,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:52,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:52,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:52,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:52,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742066_1242 (size=12301) 2024-12-09T01:41:52,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-09T01:41:52,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:52,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:52,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:52,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:52,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:52,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-09T01:41:52,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:52,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:52,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:52,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:52,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:41:52,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/1dd7dd8e6a5d4030a82e74c85720ed26 2024-12-09T01:41:52,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/be3b222bf7754177ab4ad6555847e134 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/be3b222bf7754177ab4ad6555847e134 2024-12-09T01:41:52,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/be3b222bf7754177ab4ad6555847e134, entries=250, sequenceid=469, filesize=47.5 K 2024-12-09T01:41:52,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/2b731b66a42c4cd7bd8768ff017f71c1 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/2b731b66a42c4cd7bd8768ff017f71c1 2024-12-09T01:41:52,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/2b731b66a42c4cd7bd8768ff017f71c1, entries=150, sequenceid=469, filesize=12.0 K 2024-12-09T01:41:52,529 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/1dd7dd8e6a5d4030a82e74c85720ed26 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/1dd7dd8e6a5d4030a82e74c85720ed26 2024-12-09T01:41:52,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/1dd7dd8e6a5d4030a82e74c85720ed26, entries=150, sequenceid=469, filesize=12.0 K 2024-12-09T01:41:52,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for ca70d838c08a81d31de52aceb8d05bfd in 1921ms, sequenceid=469, compaction requested=true 2024-12-09T01:41:52,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:52,535 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:52,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca70d838c08a81d31de52aceb8d05bfd:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:41:52,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:52,535 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:52,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca70d838c08a81d31de52aceb8d05bfd:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:41:52,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:52,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ca70d838c08a81d31de52aceb8d05bfd:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:41:52,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:52,536 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120525 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:52,536 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): ca70d838c08a81d31de52aceb8d05bfd/A is initiating minor compaction (all files) 2024-12-09T01:41:52,536 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca70d838c08a81d31de52aceb8d05bfd/A in TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:52,536 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/dc6f8f9ea36e4c25bd1486bbaaf5e39b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5aed1fd9c5cd422e9596bc57795f283c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/be3b222bf7754177ab4ad6555847e134] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp, totalSize=117.7 K 2024-12-09T01:41:52,537 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:52,537 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. files: [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/dc6f8f9ea36e4c25bd1486bbaaf5e39b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5aed1fd9c5cd422e9596bc57795f283c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/be3b222bf7754177ab4ad6555847e134] 2024-12-09T01:41:52,538 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:52,538 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): ca70d838c08a81d31de52aceb8d05bfd/B is initiating minor compaction (all files) 2024-12-09T01:41:52,538 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca70d838c08a81d31de52aceb8d05bfd/B in TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:52,538 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/d565b778838142ab8a07c3c64d3d04fd, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/244915fb945a4cc99a4790a47a834393, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/2b731b66a42c4cd7bd8768ff017f71c1] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp, totalSize=36.7 K 2024-12-09T01:41:52,539 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc6f8f9ea36e4c25bd1486bbaaf5e39b, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733708507909 2024-12-09T01:41:52,539 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting d565b778838142ab8a07c3c64d3d04fd, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733708507909 2024-12-09T01:41:52,539 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5aed1fd9c5cd422e9596bc57795f283c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1733708508405 2024-12-09T01:41:52,539 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 244915fb945a4cc99a4790a47a834393, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1733708508408 2024-12-09T01:41:52,539 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting be3b222bf7754177ab4ad6555847e134, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1733708509582 2024-12-09T01:41:52,540 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b731b66a42c4cd7bd8768ff017f71c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1733708510596 2024-12-09T01:41:52,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-12-09T01:41:52,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:52,547 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing ca70d838c08a81d31de52aceb8d05bfd 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T01:41:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=A 2024-12-09T01:41:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=B 2024-12-09T01:41:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=C 2024-12-09T01:41:52,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:41:52,554 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca70d838c08a81d31de52aceb8d05bfd#B#compaction#204 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:52,555 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/f0f975cc137e41c196a63538d38de631 is 50, key is test_row_0/B:col10/1733708510596/Put/seqid=0 2024-12-09T01:41:52,572 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:52,577 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120921802b10c38045a18d3cf739eb6dc741_ca70d838c08a81d31de52aceb8d05bfd store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:52,579 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120921802b10c38045a18d3cf739eb6dc741_ca70d838c08a81d31de52aceb8d05bfd, store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:52,579 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120921802b10c38045a18d3cf739eb6dc741_ca70d838c08a81d31de52aceb8d05bfd because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:52,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412094012542701d644f796ff73287d10f1b4_ca70d838c08a81d31de52aceb8d05bfd is 50, key is test_row_0/A:col10/1733708510824/Put/seqid=0 2024-12-09T01:41:52,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742067_1243 (size=13085) 2024-12-09T01:41:52,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742068_1244 (size=4469) 2024-12-09T01:41:52,600 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca70d838c08a81d31de52aceb8d05bfd#A#compaction#205 average throughput is 0.87 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:52,601 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/a50b5835ef4d41868801d4332b2ad874 is 175, key is test_row_0/A:col10/1733708510596/Put/seqid=0 2024-12-09T01:41:52,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742069_1245 (size=12454) 2024-12-09T01:41:52,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742070_1246 (size=32039) 2024-12-09T01:41:52,645 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/a50b5835ef4d41868801d4332b2ad874 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/a50b5835ef4d41868801d4332b2ad874 2024-12-09T01:41:52,652 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca70d838c08a81d31de52aceb8d05bfd/A of ca70d838c08a81d31de52aceb8d05bfd into a50b5835ef4d41868801d4332b2ad874(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:52,652 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:52,652 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., storeName=ca70d838c08a81d31de52aceb8d05bfd/A, priority=13, startTime=1733708512535; duration=0sec 2024-12-09T01:41:52,652 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:41:52,652 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca70d838c08a81d31de52aceb8d05bfd:A 2024-12-09T01:41:52,652 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:41:52,654 INFO [Thread-746 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-746. 2024-12-09T01:41:52,654 DEBUG [Thread-746 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:41:52,654 DEBUG [Thread-746 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:52,654 DEBUG [Thread-746 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:52,654 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:41:52,656 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:41:52,656 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): ca70d838c08a81d31de52aceb8d05bfd/C is initiating minor compaction (all files) 2024-12-09T01:41:52,656 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ca70d838c08a81d31de52aceb8d05bfd/C in TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:52,657 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/87c76479d2254e109b83f287f43b1603, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/6202c95b8ecc4754ad243435ca018840, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/1dd7dd8e6a5d4030a82e74c85720ed26] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp, totalSize=36.7 K 2024-12-09T01:41:52,657 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87c76479d2254e109b83f287f43b1603, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1733708507909 2024-12-09T01:41:52,657 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6202c95b8ecc4754ad243435ca018840, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1733708508408 2024-12-09T01:41:52,658 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1dd7dd8e6a5d4030a82e74c85720ed26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=469, earliestPutTs=1733708510596 2024-12-09T01:41:52,661 INFO [Thread-744 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-744. 2024-12-09T01:41:52,661 DEBUG [Thread-744 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:41:52,661 DEBUG [Thread-744 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:52,661 DEBUG [Thread-744 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:52,661 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:41:52,664 INFO [Thread-748 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-748. 2024-12-09T01:41:52,664 DEBUG [Thread-748 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:41:52,664 DEBUG [Thread-748 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:52,664 DEBUG [Thread-748 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:52,664 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:41:52,666 INFO [Thread-750 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-750. 2024-12-09T01:41:52,666 DEBUG [Thread-750 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:41:52,666 DEBUG [Thread-750 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:52,666 DEBUG [Thread-750 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:52,666 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:41:52,666 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ca70d838c08a81d31de52aceb8d05bfd#C#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:41:52,667 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/1dec3ad850434283bba2c3c45ccbdf43 is 50, key is test_row_0/C:col10/1733708510596/Put/seqid=0 2024-12-09T01:41:52,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742071_1247 (size=13085) 2024-12-09T01:41:52,990 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/f0f975cc137e41c196a63538d38de631 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/f0f975cc137e41c196a63538d38de631 2024-12-09T01:41:52,998 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca70d838c08a81d31de52aceb8d05bfd/B of ca70d838c08a81d31de52aceb8d05bfd into f0f975cc137e41c196a63538d38de631(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:52,998 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:52,999 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., storeName=ca70d838c08a81d31de52aceb8d05bfd/B, priority=13, startTime=1733708512535; duration=0sec 2024-12-09T01:41:52,999 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:52,999 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca70d838c08a81d31de52aceb8d05bfd:B 2024-12-09T01:41:53,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:53,013 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412094012542701d644f796ff73287d10f1b4_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094012542701d644f796ff73287d10f1b4_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:53,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/a6464db2153747edacc944676fa59864, store: [table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:53,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/a6464db2153747edacc944676fa59864 is 175, key is test_row_0/A:col10/1733708510824/Put/seqid=0 2024-12-09T01:41:53,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:53,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. as already flushing 2024-12-09T01:41:53,026 INFO [Thread-735 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-735. 2024-12-09T01:41:53,026 DEBUG [Thread-735 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:41:53,026 DEBUG [Thread-735 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:53,027 INFO [Thread-733 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-733. 2024-12-09T01:41:53,027 DEBUG [Thread-733 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:41:53,027 INFO [Thread-741 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-741. 2024-12-09T01:41:53,027 DEBUG [Thread-735 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:53,027 DEBUG [Thread-733 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:53,027 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:41:53,027 DEBUG [Thread-741 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:41:53,027 DEBUG [Thread-741 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:53,027 DEBUG [Thread-733 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:53,027 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:41:53,027 DEBUG [Thread-741 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:53,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742072_1248 (size=31255) 2024-12-09T01:41:53,028 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:41:53,036 INFO [Thread-737 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-737. 2024-12-09T01:41:53,036 DEBUG [Thread-737 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:41:53,036 DEBUG [Thread-737 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:53,036 DEBUG [Thread-737 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:53,036 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:41:53,078 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/1dec3ad850434283bba2c3c45ccbdf43 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/1dec3ad850434283bba2c3c45ccbdf43 2024-12-09T01:41:53,083 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ca70d838c08a81d31de52aceb8d05bfd/C of ca70d838c08a81d31de52aceb8d05bfd into 1dec3ad850434283bba2c3c45ccbdf43(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:41:53,083 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:53,083 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., storeName=ca70d838c08a81d31de52aceb8d05bfd/C, priority=13, startTime=1733708512535; duration=0sec 2024-12-09T01:41:53,083 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:41:53,083 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ca70d838c08a81d31de52aceb8d05bfd:C 2024-12-09T01:41:53,428 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=505, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/a6464db2153747edacc944676fa59864 2024-12-09T01:41:53,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/71c5100f3a2b491f9c827d73ebb59210 is 50, key is test_row_0/B:col10/1733708510824/Put/seqid=0 2024-12-09T01:41:53,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742073_1249 (size=12301) 2024-12-09T01:41:53,841 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/71c5100f3a2b491f9c827d73ebb59210 2024-12-09T01:41:53,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/cd5155ec2f074d039e7c03e557d50e31 is 50, key is test_row_0/C:col10/1733708510824/Put/seqid=0 2024-12-09T01:41:53,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742074_1250 (size=12301) 2024-12-09T01:41:54,055 INFO [Thread-739 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-739. 2024-12-09T01:41:54,055 DEBUG [Thread-739 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:41:54,055 DEBUG [Thread-739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:54,055 DEBUG [Thread-739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:54,055 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:41:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-09T01:41:54,258 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=505 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/cd5155ec2f074d039e7c03e557d50e31 2024-12-09T01:41:54,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/a6464db2153747edacc944676fa59864 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/a6464db2153747edacc944676fa59864 2024-12-09T01:41:54,268 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/a6464db2153747edacc944676fa59864, entries=150, sequenceid=505, filesize=30.5 K 2024-12-09T01:41:54,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/71c5100f3a2b491f9c827d73ebb59210 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/71c5100f3a2b491f9c827d73ebb59210 2024-12-09T01:41:54,273 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/71c5100f3a2b491f9c827d73ebb59210, entries=150, sequenceid=505, filesize=12.0 K 2024-12-09T01:41:54,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/cd5155ec2f074d039e7c03e557d50e31 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/cd5155ec2f074d039e7c03e557d50e31 2024-12-09T01:41:54,278 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/cd5155ec2f074d039e7c03e557d50e31, entries=150, sequenceid=505, filesize=12.0 K 2024-12-09T01:41:54,279 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=33.54 KB/34350 for ca70d838c08a81d31de52aceb8d05bfd in 1732ms, sequenceid=505, compaction requested=false 2024-12-09T01:41:54,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for ca70d838c08a81d31de52aceb8d05bfd: 2024-12-09T01:41:54,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:54,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-12-09T01:41:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-12-09T01:41:54,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-12-09T01:41:54,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.3610 sec 2024-12-09T01:41:54,284 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 4.3670 sec 2024-12-09T01:41:58,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-09T01:41:58,114 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:41:58,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T01:41:58,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-12-09T01:41:58,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 91 2024-12-09T01:41:58,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-12-09T01:41:58,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-12-09T01:41:58,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 78 2024-12-09T01:41:58,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T01:41:58,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2222 2024-12-09T01:41:58,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2284 2024-12-09T01:41:58,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T01:41:58,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1019 2024-12-09T01:41:58,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3051 rows 2024-12-09T01:41:58,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1019 2024-12-09T01:41:58,114 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3053 rows 2024-12-09T01:41:58,115 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T01:41:58,115 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T01:41:58,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:58,115 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:41:58,115 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:41:58,116 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T01:41:58,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T01:41:58,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-09T01:41:58,119 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708518119"}]},"ts":"1733708518119"} 2024-12-09T01:41:58,121 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T01:41:58,121 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T01:41:58,121 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T01:41:58,123 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca70d838c08a81d31de52aceb8d05bfd, UNASSIGN}] 2024-12-09T01:41:58,124 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca70d838c08a81d31de52aceb8d05bfd, UNASSIGN 2024-12-09T01:41:58,124 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=ca70d838c08a81d31de52aceb8d05bfd, regionState=CLOSING, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:41:58,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca70d838c08a81d31de52aceb8d05bfd, UNASSIGN because future has completed 2024-12-09T01:41:58,126 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T01:41:58,126 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943}] 2024-12-09T01:41:58,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-09T01:41:58,278 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:58,278 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T01:41:58,278 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing ca70d838c08a81d31de52aceb8d05bfd, disabling compactions & flushes 2024-12-09T01:41:58,278 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:58,278 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:58,278 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. after waiting 0 ms 2024-12-09T01:41:58,278 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:41:58,278 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing ca70d838c08a81d31de52aceb8d05bfd 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-09T01:41:58,278 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=A 2024-12-09T01:41:58,279 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:58,279 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=B 2024-12-09T01:41:58,279 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:58,279 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ca70d838c08a81d31de52aceb8d05bfd, store=C 2024-12-09T01:41:58,279 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:41:58,286 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120940862d8eacdb4a61b25bfd1ec0ceb2c9_ca70d838c08a81d31de52aceb8d05bfd is 50, key is test_row_1/A:col10/1733708514054/Put/seqid=0 2024-12-09T01:41:58,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742075_1251 (size=9914) 2024-12-09T01:41:58,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-09T01:41:58,690 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:41:58,695 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120940862d8eacdb4a61b25bfd1ec0ceb2c9_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120940862d8eacdb4a61b25bfd1ec0ceb2c9_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:41:58,696 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/7693ab48e08a4b83bcd841923f21ecea, store: [table=TestAcidGuarantees family=A region=ca70d838c08a81d31de52aceb8d05bfd] 2024-12-09T01:41:58,696 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/7693ab48e08a4b83bcd841923f21ecea is 175, key is test_row_1/A:col10/1733708514054/Put/seqid=0 2024-12-09T01:41:58,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742076_1252 (size=22561) 2024-12-09T01:41:58,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-09T01:41:59,101 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=516, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/7693ab48e08a4b83bcd841923f21ecea 2024-12-09T01:41:59,108 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/65a95845a1d84d94a9bfde0278be94a6 is 50, key is test_row_1/B:col10/1733708514054/Put/seqid=0 2024-12-09T01:41:59,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742077_1253 (size=9857) 2024-12-09T01:41:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-09T01:41:59,513 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/65a95845a1d84d94a9bfde0278be94a6 2024-12-09T01:41:59,520 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/784d2ab03eba4f4a91119535612e71dc is 50, key is test_row_1/C:col10/1733708514054/Put/seqid=0 2024-12-09T01:41:59,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742078_1254 (size=9857) 2024-12-09T01:41:59,925 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=516 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/784d2ab03eba4f4a91119535612e71dc 2024-12-09T01:41:59,929 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/A/7693ab48e08a4b83bcd841923f21ecea as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/7693ab48e08a4b83bcd841923f21ecea 2024-12-09T01:41:59,933 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/7693ab48e08a4b83bcd841923f21ecea, entries=100, sequenceid=516, filesize=22.0 K 2024-12-09T01:41:59,934 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/B/65a95845a1d84d94a9bfde0278be94a6 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/65a95845a1d84d94a9bfde0278be94a6 2024-12-09T01:41:59,937 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/65a95845a1d84d94a9bfde0278be94a6, entries=100, sequenceid=516, filesize=9.6 K 2024-12-09T01:41:59,938 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/.tmp/C/784d2ab03eba4f4a91119535612e71dc as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/784d2ab03eba4f4a91119535612e71dc 2024-12-09T01:41:59,942 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/784d2ab03eba4f4a91119535612e71dc, entries=100, sequenceid=516, filesize=9.6 K 2024-12-09T01:41:59,943 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ca70d838c08a81d31de52aceb8d05bfd in 1664ms, sequenceid=516, compaction requested=true 2024-12-09T01:41:59,944 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/ba7b6217a631419cbb8bba6eff13a391, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5dd60be42f644445a0e6b7e4e048121a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5593f999b32f48b29e35237f114a53ee, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/6d7f0f5586da4c4caac78312da9603ad, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5e8a9207ef0f419eb37279db59135c52, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/2218d6fe5a074f2bb00d26523c6070b1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/e46076ed4ac24e69b86ff22df55c85de, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/3dd7d191973c4e0586b71f301a71ad76, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/06cfac23d163446381785a5ab72885ee, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/8ebaf5de9e13425d8ed67de66428e5f6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/965c73ab899e4ff8b8cee96d305300db, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/bcde21feb8304a2a849edc29a77caa78, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/dfc75619368649b8a3f5dc617e5a2d0a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/02757be7af2e4798b052c62047faede5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/2910f7498c304b51aeae7b68ba606e04, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/b95b16ffc35c438e9a3fd2b31feb3498, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/430549c873a649fbace40647a3fee393, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/ca9d8872599a420aabd3dbc8cf6202d3, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/4f6703a0d54b4726bf693a9de6a9854c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/ce041a9c34484d448d056aa8fed32eb9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/dc6f8f9ea36e4c25bd1486bbaaf5e39b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5aed1fd9c5cd422e9596bc57795f283c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/be3b222bf7754177ab4ad6555847e134] to archive 2024-12-09T01:41:59,945 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T01:41:59,946 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/ba7b6217a631419cbb8bba6eff13a391 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/ba7b6217a631419cbb8bba6eff13a391 2024-12-09T01:41:59,947 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5dd60be42f644445a0e6b7e4e048121a to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5dd60be42f644445a0e6b7e4e048121a 2024-12-09T01:41:59,948 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5593f999b32f48b29e35237f114a53ee to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5593f999b32f48b29e35237f114a53ee 2024-12-09T01:41:59,949 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/6d7f0f5586da4c4caac78312da9603ad to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/6d7f0f5586da4c4caac78312da9603ad 2024-12-09T01:41:59,950 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5e8a9207ef0f419eb37279db59135c52 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5e8a9207ef0f419eb37279db59135c52 2024-12-09T01:41:59,951 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/2218d6fe5a074f2bb00d26523c6070b1 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/2218d6fe5a074f2bb00d26523c6070b1 2024-12-09T01:41:59,952 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/e46076ed4ac24e69b86ff22df55c85de to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/e46076ed4ac24e69b86ff22df55c85de 2024-12-09T01:41:59,953 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/3dd7d191973c4e0586b71f301a71ad76 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/3dd7d191973c4e0586b71f301a71ad76 2024-12-09T01:41:59,954 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/06cfac23d163446381785a5ab72885ee to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/06cfac23d163446381785a5ab72885ee 2024-12-09T01:41:59,955 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/8ebaf5de9e13425d8ed67de66428e5f6 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/8ebaf5de9e13425d8ed67de66428e5f6 2024-12-09T01:41:59,956 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/965c73ab899e4ff8b8cee96d305300db to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/965c73ab899e4ff8b8cee96d305300db 2024-12-09T01:41:59,957 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/bcde21feb8304a2a849edc29a77caa78 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/bcde21feb8304a2a849edc29a77caa78 2024-12-09T01:41:59,958 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/dfc75619368649b8a3f5dc617e5a2d0a to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/dfc75619368649b8a3f5dc617e5a2d0a 2024-12-09T01:41:59,959 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/02757be7af2e4798b052c62047faede5 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/02757be7af2e4798b052c62047faede5 2024-12-09T01:41:59,960 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/2910f7498c304b51aeae7b68ba606e04 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/2910f7498c304b51aeae7b68ba606e04 2024-12-09T01:41:59,962 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/b95b16ffc35c438e9a3fd2b31feb3498 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/b95b16ffc35c438e9a3fd2b31feb3498 2024-12-09T01:41:59,963 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/430549c873a649fbace40647a3fee393 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/430549c873a649fbace40647a3fee393 2024-12-09T01:41:59,964 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/ca9d8872599a420aabd3dbc8cf6202d3 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/ca9d8872599a420aabd3dbc8cf6202d3 2024-12-09T01:41:59,965 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/4f6703a0d54b4726bf693a9de6a9854c to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/4f6703a0d54b4726bf693a9de6a9854c 2024-12-09T01:41:59,966 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/ce041a9c34484d448d056aa8fed32eb9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/ce041a9c34484d448d056aa8fed32eb9 2024-12-09T01:41:59,967 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/dc6f8f9ea36e4c25bd1486bbaaf5e39b to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/dc6f8f9ea36e4c25bd1486bbaaf5e39b 2024-12-09T01:41:59,969 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5aed1fd9c5cd422e9596bc57795f283c to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/5aed1fd9c5cd422e9596bc57795f283c 2024-12-09T01:41:59,970 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/be3b222bf7754177ab4ad6555847e134 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/be3b222bf7754177ab4ad6555847e134 2024-12-09T01:41:59,971 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/5f25f9dd97e6408ca6ba863add174329, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/9ef94889f8fe418da1d7390e0898ff51, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/0632f16c711940a381009f2a58c39ba5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/f00ec6e685d74a048bd6d1c13429d4e0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/9fbb60bc2e2543cc8fbc1205c6ed7032, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/f107f57fcfec48c1bbda31d721709311, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/25cb591e9ccb488a8e8e39e94a34deff, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/42605c26f12a4007a7c2780b5607b10d, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/8e49142163884babaf6c347f74a150c0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/81ea69653e9d4cbfa17ca12d019101c1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/7987c2d0270644639ea9af37de2a07d9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/1990232f30f148e6bb0b078597f772bf, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/ac3ec016320e4668b890171699189d2e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/2aabcabb0c404157ad542b10539ebc38, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/b3572547ce5e483e8f2e10d55b195254, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/4d1fcbb4bc8743ae82d071e9020763fe, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/7fbfdc7d6e134b6aa2dddb430fac0f31, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/35b21c2ae8dd47fd8c05c98b7f7c369f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/6fb92d81cb554132aa5e805f3986aad3, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/d565b778838142ab8a07c3c64d3d04fd, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/5f6026d1447d4f1bb0d8b78595ed9962, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/244915fb945a4cc99a4790a47a834393, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/2b731b66a42c4cd7bd8768ff017f71c1] to archive 2024-12-09T01:41:59,972 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T01:41:59,974 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/5f25f9dd97e6408ca6ba863add174329 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/5f25f9dd97e6408ca6ba863add174329 2024-12-09T01:41:59,975 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/9ef94889f8fe418da1d7390e0898ff51 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/9ef94889f8fe418da1d7390e0898ff51 2024-12-09T01:41:59,976 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/0632f16c711940a381009f2a58c39ba5 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/0632f16c711940a381009f2a58c39ba5 2024-12-09T01:41:59,977 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/f00ec6e685d74a048bd6d1c13429d4e0 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/f00ec6e685d74a048bd6d1c13429d4e0 2024-12-09T01:41:59,978 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/9fbb60bc2e2543cc8fbc1205c6ed7032 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/9fbb60bc2e2543cc8fbc1205c6ed7032 2024-12-09T01:41:59,979 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/f107f57fcfec48c1bbda31d721709311 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/f107f57fcfec48c1bbda31d721709311 2024-12-09T01:41:59,980 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/25cb591e9ccb488a8e8e39e94a34deff to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/25cb591e9ccb488a8e8e39e94a34deff 2024-12-09T01:41:59,981 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/42605c26f12a4007a7c2780b5607b10d to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/42605c26f12a4007a7c2780b5607b10d 2024-12-09T01:41:59,983 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/8e49142163884babaf6c347f74a150c0 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/8e49142163884babaf6c347f74a150c0 2024-12-09T01:41:59,984 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/81ea69653e9d4cbfa17ca12d019101c1 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/81ea69653e9d4cbfa17ca12d019101c1 2024-12-09T01:41:59,985 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/7987c2d0270644639ea9af37de2a07d9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/7987c2d0270644639ea9af37de2a07d9 2024-12-09T01:41:59,986 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/1990232f30f148e6bb0b078597f772bf to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/1990232f30f148e6bb0b078597f772bf 2024-12-09T01:41:59,987 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/ac3ec016320e4668b890171699189d2e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/ac3ec016320e4668b890171699189d2e 2024-12-09T01:41:59,988 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/2aabcabb0c404157ad542b10539ebc38 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/2aabcabb0c404157ad542b10539ebc38 2024-12-09T01:41:59,989 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/b3572547ce5e483e8f2e10d55b195254 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/b3572547ce5e483e8f2e10d55b195254 2024-12-09T01:41:59,990 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/4d1fcbb4bc8743ae82d071e9020763fe to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/4d1fcbb4bc8743ae82d071e9020763fe 2024-12-09T01:41:59,991 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/7fbfdc7d6e134b6aa2dddb430fac0f31 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/7fbfdc7d6e134b6aa2dddb430fac0f31 2024-12-09T01:41:59,992 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/35b21c2ae8dd47fd8c05c98b7f7c369f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/35b21c2ae8dd47fd8c05c98b7f7c369f 2024-12-09T01:41:59,993 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/6fb92d81cb554132aa5e805f3986aad3 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/6fb92d81cb554132aa5e805f3986aad3 2024-12-09T01:41:59,994 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/d565b778838142ab8a07c3c64d3d04fd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/d565b778838142ab8a07c3c64d3d04fd 2024-12-09T01:41:59,995 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/5f6026d1447d4f1bb0d8b78595ed9962 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/5f6026d1447d4f1bb0d8b78595ed9962 2024-12-09T01:41:59,996 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/244915fb945a4cc99a4790a47a834393 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/244915fb945a4cc99a4790a47a834393 2024-12-09T01:41:59,998 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/2b731b66a42c4cd7bd8768ff017f71c1 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/2b731b66a42c4cd7bd8768ff017f71c1 2024-12-09T01:41:59,999 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/6f37d1eb87dc4f01ba589628f3007560, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/a88fb51330f244b795bbf1832d831013, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/624b725017ee4eca90e1c5900fd4434f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/e160020cd52148889093e85ee8fb8227, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/067515fc844d4d7cada2f9ec351bf953, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/46b10ff016ec43e992e4fc0e882d00b4, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/d26094c2aefb498eb0ccfa5ee346227a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/a1af3b6d6c5a4d5d8f86e063a1568f82, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/d6b41cafe8ff4ddbabe30b2409988408, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/bacfbd84e03141c8b3b29693f4592e0b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/0baaa9f8e0df44fa87619e1274b3901e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/756da9c587054ffdbb6b6dcb397b9928, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/f7a2fa0fcd5b4e82afe440793241495c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/d971de3c5b13429b8128b443ee9c488e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/8f05ad379b1f4b23a452804c3457faa7, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/d8cb9817a0a4475cb95aaa651e2c7901, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/6fba8204ee92413886a1e20a3fb8b317, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/b298c2e6b13343d7881ba15ec47d5d60, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/874d64dbf6ae4962affc9552f6fe3099, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/87c76479d2254e109b83f287f43b1603, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/0b7df81e659f4d70b83b6c595440000e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/6202c95b8ecc4754ad243435ca018840, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/1dd7dd8e6a5d4030a82e74c85720ed26] to archive 2024-12-09T01:42:00,000 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T01:42:00,002 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/6f37d1eb87dc4f01ba589628f3007560 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/6f37d1eb87dc4f01ba589628f3007560 2024-12-09T01:42:00,003 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/a88fb51330f244b795bbf1832d831013 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/a88fb51330f244b795bbf1832d831013 2024-12-09T01:42:00,004 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/624b725017ee4eca90e1c5900fd4434f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/624b725017ee4eca90e1c5900fd4434f 2024-12-09T01:42:00,006 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/e160020cd52148889093e85ee8fb8227 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/e160020cd52148889093e85ee8fb8227 2024-12-09T01:42:00,007 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/067515fc844d4d7cada2f9ec351bf953 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/067515fc844d4d7cada2f9ec351bf953 2024-12-09T01:42:00,008 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/46b10ff016ec43e992e4fc0e882d00b4 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/46b10ff016ec43e992e4fc0e882d00b4 2024-12-09T01:42:00,009 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/d26094c2aefb498eb0ccfa5ee346227a to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/d26094c2aefb498eb0ccfa5ee346227a 2024-12-09T01:42:00,011 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/a1af3b6d6c5a4d5d8f86e063a1568f82 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/a1af3b6d6c5a4d5d8f86e063a1568f82 2024-12-09T01:42:00,012 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/d6b41cafe8ff4ddbabe30b2409988408 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/d6b41cafe8ff4ddbabe30b2409988408 2024-12-09T01:42:00,013 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/bacfbd84e03141c8b3b29693f4592e0b to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/bacfbd84e03141c8b3b29693f4592e0b 2024-12-09T01:42:00,015 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/0baaa9f8e0df44fa87619e1274b3901e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/0baaa9f8e0df44fa87619e1274b3901e 2024-12-09T01:42:00,016 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/756da9c587054ffdbb6b6dcb397b9928 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/756da9c587054ffdbb6b6dcb397b9928 2024-12-09T01:42:00,017 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/f7a2fa0fcd5b4e82afe440793241495c to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/f7a2fa0fcd5b4e82afe440793241495c 2024-12-09T01:42:00,018 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/d971de3c5b13429b8128b443ee9c488e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/d971de3c5b13429b8128b443ee9c488e 2024-12-09T01:42:00,019 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/8f05ad379b1f4b23a452804c3457faa7 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/8f05ad379b1f4b23a452804c3457faa7 2024-12-09T01:42:00,020 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/d8cb9817a0a4475cb95aaa651e2c7901 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/d8cb9817a0a4475cb95aaa651e2c7901 2024-12-09T01:42:00,021 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/6fba8204ee92413886a1e20a3fb8b317 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/6fba8204ee92413886a1e20a3fb8b317 2024-12-09T01:42:00,022 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/b298c2e6b13343d7881ba15ec47d5d60 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/b298c2e6b13343d7881ba15ec47d5d60 2024-12-09T01:42:00,023 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/874d64dbf6ae4962affc9552f6fe3099 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/874d64dbf6ae4962affc9552f6fe3099 2024-12-09T01:42:00,024 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/87c76479d2254e109b83f287f43b1603 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/87c76479d2254e109b83f287f43b1603 2024-12-09T01:42:00,025 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/0b7df81e659f4d70b83b6c595440000e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/0b7df81e659f4d70b83b6c595440000e 2024-12-09T01:42:00,026 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/6202c95b8ecc4754ad243435ca018840 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/6202c95b8ecc4754ad243435ca018840 2024-12-09T01:42:00,027 DEBUG [StoreCloser-TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/1dd7dd8e6a5d4030a82e74c85720ed26 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/1dd7dd8e6a5d4030a82e74c85720ed26 2024-12-09T01:42:00,032 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/recovered.edits/519.seqid, newMaxSeqId=519, maxSeqId=4 2024-12-09T01:42:00,033 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. 2024-12-09T01:42:00,033 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for ca70d838c08a81d31de52aceb8d05bfd: Waiting for close lock at 1733708518278Running coprocessor pre-close hooks at 1733708518278Disabling compacts and flushes for region at 1733708518278Disabling writes for close at 1733708518278Obtaining lock to block concurrent updates at 1733708518278Preparing flush snapshotting stores in ca70d838c08a81d31de52aceb8d05bfd at 1733708518278Finished memstore snapshotting TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733708518279 (+1 ms)Flushing stores of TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd. at 1733708518279Flushing ca70d838c08a81d31de52aceb8d05bfd/A: creating writer at 1733708518279Flushing mob file ca70d838c08a81d31de52aceb8d05bfd/A: appending metadata at 1733708518285 (+6 ms)Flushing mob file ca70d838c08a81d31de52aceb8d05bfd/A: closing flushed file at 1733708518285Flushing ca70d838c08a81d31de52aceb8d05bfd/A: appending metadata at 1733708518696 (+411 ms)Flushing ca70d838c08a81d31de52aceb8d05bfd/A: closing flushed file at 1733708518696Flushing ca70d838c08a81d31de52aceb8d05bfd/B: creating writer at 1733708519105 (+409 ms)Flushing ca70d838c08a81d31de52aceb8d05bfd/B: appending metadata at 1733708519108 (+3 ms)Flushing ca70d838c08a81d31de52aceb8d05bfd/B: closing flushed file at 1733708519108Flushing ca70d838c08a81d31de52aceb8d05bfd/C: creating writer at 1733708519517 (+409 ms)Flushing ca70d838c08a81d31de52aceb8d05bfd/C: appending metadata at 1733708519519 (+2 ms)Flushing ca70d838c08a81d31de52aceb8d05bfd/C: closing flushed file at 1733708519519Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51be9d34: reopening flushed file at 1733708519929 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66d1dfe7: reopening flushed file at 1733708519933 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a2c8cd6: reopening flushed file at 1733708519938 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for ca70d838c08a81d31de52aceb8d05bfd in 1664ms, sequenceid=516, compaction requested=true at 1733708519943 (+5 ms)Writing region close event to WAL at 1733708520029 (+86 ms)Running coprocessor post-close hooks at 1733708520033 (+4 ms)Closed at 1733708520033 2024-12-09T01:42:00,036 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:42:00,036 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=ca70d838c08a81d31de52aceb8d05bfd, regionState=CLOSED 2024-12-09T01:42:00,038 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 because future has completed 2024-12-09T01:42:00,040 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-12-09T01:42:00,040 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure ca70d838c08a81d31de52aceb8d05bfd, server=ef6f18c58dc9,38627,1733708453943 in 1.9120 sec 2024-12-09T01:42:00,041 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-09T01:42:00,041 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ca70d838c08a81d31de52aceb8d05bfd, UNASSIGN in 1.9170 sec 2024-12-09T01:42:00,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-12-09T01:42:00,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9210 sec 2024-12-09T01:42:00,045 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708520045"}]},"ts":"1733708520045"} 2024-12-09T01:42:00,046 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T01:42:00,046 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T01:42:00,048 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9310 sec 2024-12-09T01:42:00,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-12-09T01:42:00,264 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:00,265 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T01:42:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:00,266 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:00,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T01:42:00,267 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:00,270 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:42:00,272 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A, FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B, FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C, FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/recovered.edits] 2024-12-09T01:42:00,274 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/7693ab48e08a4b83bcd841923f21ecea to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/7693ab48e08a4b83bcd841923f21ecea 2024-12-09T01:42:00,275 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/a50b5835ef4d41868801d4332b2ad874 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/a50b5835ef4d41868801d4332b2ad874 2024-12-09T01:42:00,276 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/a6464db2153747edacc944676fa59864 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/A/a6464db2153747edacc944676fa59864 2024-12-09T01:42:00,278 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/65a95845a1d84d94a9bfde0278be94a6 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/65a95845a1d84d94a9bfde0278be94a6 2024-12-09T01:42:00,279 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/71c5100f3a2b491f9c827d73ebb59210 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/71c5100f3a2b491f9c827d73ebb59210 2024-12-09T01:42:00,280 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/f0f975cc137e41c196a63538d38de631 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/B/f0f975cc137e41c196a63538d38de631 2024-12-09T01:42:00,282 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/1dec3ad850434283bba2c3c45ccbdf43 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/1dec3ad850434283bba2c3c45ccbdf43 2024-12-09T01:42:00,283 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/784d2ab03eba4f4a91119535612e71dc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/784d2ab03eba4f4a91119535612e71dc 2024-12-09T01:42:00,284 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/cd5155ec2f074d039e7c03e557d50e31 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/C/cd5155ec2f074d039e7c03e557d50e31 2024-12-09T01:42:00,286 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/recovered.edits/519.seqid to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd/recovered.edits/519.seqid 2024-12-09T01:42:00,287 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:42:00,287 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T01:42:00,287 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T01:42:00,288 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-09T01:42:00,291 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120913980a38463a407d92020113c17d31b2_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120913980a38463a407d92020113c17d31b2_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:42:00,292 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091f26ceb4d5a944bba39629051fafce6c_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412091f26ceb4d5a944bba39629051fafce6c_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:42:00,294 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209215ece1774e64be1a6750290d7557c9e_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209215ece1774e64be1a6750290d7557c9e_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:42:00,295 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092a01d397078649a8a1a7d41cac2f2b5f_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092a01d397078649a8a1a7d41cac2f2b5f_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:42:00,296 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120935aca110df764a39b22ff8968d25cd4e_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120935aca110df764a39b22ff8968d25cd4e_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:42:00,297 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094012542701d644f796ff73287d10f1b4_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094012542701d644f796ff73287d10f1b4_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:42:00,298 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120940862d8eacdb4a61b25bfd1ec0ceb2c9_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120940862d8eacdb4a61b25bfd1ec0ceb2c9_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:42:00,299 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095c529ca6567b447992673e53c5a89fd1_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412095c529ca6567b447992673e53c5a89fd1_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:42:00,300 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120971b8cd797c3246ee95fbc9154328e8d2_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120971b8cd797c3246ee95fbc9154328e8d2_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:42:00,301 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120972f078f20e6c40179f3204833c29dd2b_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120972f078f20e6c40179f3204833c29dd2b_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:42:00,302 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097e193e11367c429bbc365eeb08da5c98_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097e193e11367c429bbc365eeb08da5c98_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:42:00,303 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209898ca7079ef64a7cac2184427b88cb33_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209898ca7079ef64a7cac2184427b88cb33_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:42:00,304 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098a5010b89c1b45aea67369497ac8a664_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098a5010b89c1b45aea67369497ac8a664_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:42:00,305 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098cd1031959814fdb8059c390182a732f_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098cd1031959814fdb8059c390182a732f_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:42:00,307 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099b0ee88ff5c54d0ebf26d4276be3c8e4_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099b0ee88ff5c54d0ebf26d4276be3c8e4_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:42:00,308 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b9eb87243bce42848d328656bab97f26_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b9eb87243bce42848d328656bab97f26_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:42:00,309 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d291d43f26454961b80663e0422650c6_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d291d43f26454961b80663e0422650c6_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:42:00,310 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f545d190090d499eb6d138c51e963927_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f545d190090d499eb6d138c51e963927_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:42:00,311 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f546b7804a724b21a1d4c67f0b6b478e_ca70d838c08a81d31de52aceb8d05bfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f546b7804a724b21a1d4c67f0b6b478e_ca70d838c08a81d31de52aceb8d05bfd 2024-12-09T01:42:00,311 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T01:42:00,314 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:00,316 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T01:42:00,319 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T01:42:00,320 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:00,321 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T01:42:00,321 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733708520321"}]},"ts":"9223372036854775807"} 2024-12-09T01:42:00,323 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T01:42:00,339 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ca70d838c08a81d31de52aceb8d05bfd, NAME => 'TestAcidGuarantees,,1733708489849.ca70d838c08a81d31de52aceb8d05bfd.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T01:42:00,339 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T01:42:00,339 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733708520339"}]},"ts":"9223372036854775807"} 2024-12-09T01:42:00,342 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T01:42:00,343 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:00,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 78 msec 2024-12-09T01:42:00,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-12-09T01:42:00,374 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T01:42:00,376 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:00,383 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 247) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/cluster_fb15f3cb-c4dd-1da4-630e-097361e28d71/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/cluster_fb15f3cb-c4dd-1da4-630e-097361e28d71/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1866703801_22 at /127.0.0.1:44646 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=575 (was 487) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=12130 (was 12468) 2024-12-09T01:42:00,392 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=575, ProcessCount=11, AvailableMemoryMB=12129 2024-12-09T01:42:00,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T01:42:00,394 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:42:00,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:00,395 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T01:42:00,396 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:00,396 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-12-09T01:42:00,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-09T01:42:00,396 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T01:42:00,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742079_1255 (size=1061) 2024-12-09T01:42:00,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-09T01:42:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-09T01:42:00,804 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb 2024-12-09T01:42:00,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742080_1256 (size=53) 2024-12-09T01:42:01,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-09T01:42:01,212 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:42:01,213 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing bb023e4ee9cc2de03187f3dff0a271ab, disabling compactions & flushes 2024-12-09T01:42:01,213 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:01,213 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:01,213 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. after waiting 0 ms 2024-12-09T01:42:01,213 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:01,213 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:01,213 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for bb023e4ee9cc2de03187f3dff0a271ab: Waiting for close lock at 1733708521213Disabling compacts and flushes for region at 1733708521213Disabling writes for close at 1733708521213Writing region close event to WAL at 1733708521213Closed at 1733708521213 2024-12-09T01:42:01,214 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T01:42:01,214 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733708521214"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733708521214"}]},"ts":"1733708521214"} 2024-12-09T01:42:01,216 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T01:42:01,217 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T01:42:01,217 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708521217"}]},"ts":"1733708521217"} 2024-12-09T01:42:01,218 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T01:42:01,218 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bb023e4ee9cc2de03187f3dff0a271ab, ASSIGN}] 2024-12-09T01:42:01,219 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bb023e4ee9cc2de03187f3dff0a271ab, ASSIGN 2024-12-09T01:42:01,220 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bb023e4ee9cc2de03187f3dff0a271ab, ASSIGN; state=OFFLINE, location=ef6f18c58dc9,38627,1733708453943; forceNewPlan=false, retain=false 2024-12-09T01:42:01,371 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=bb023e4ee9cc2de03187f3dff0a271ab, regionState=OPENING, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:01,373 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bb023e4ee9cc2de03187f3dff0a271ab, ASSIGN because future has completed 2024-12-09T01:42:01,373 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943}] 2024-12-09T01:42:01,528 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:01,528 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:42:01,528 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:01,529 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:42:01,529 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:01,529 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:01,530 INFO [StoreOpener-bb023e4ee9cc2de03187f3dff0a271ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:01,531 INFO [StoreOpener-bb023e4ee9cc2de03187f3dff0a271ab-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:42:01,531 INFO [StoreOpener-bb023e4ee9cc2de03187f3dff0a271ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb023e4ee9cc2de03187f3dff0a271ab columnFamilyName A 2024-12-09T01:42:01,531 DEBUG [StoreOpener-bb023e4ee9cc2de03187f3dff0a271ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:01,532 INFO [StoreOpener-bb023e4ee9cc2de03187f3dff0a271ab-1 {}] regionserver.HStore(327): Store=bb023e4ee9cc2de03187f3dff0a271ab/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:42:01,532 INFO [StoreOpener-bb023e4ee9cc2de03187f3dff0a271ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:01,533 INFO [StoreOpener-bb023e4ee9cc2de03187f3dff0a271ab-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:42:01,533 INFO [StoreOpener-bb023e4ee9cc2de03187f3dff0a271ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb023e4ee9cc2de03187f3dff0a271ab columnFamilyName B 2024-12-09T01:42:01,533 DEBUG [StoreOpener-bb023e4ee9cc2de03187f3dff0a271ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:01,533 INFO [StoreOpener-bb023e4ee9cc2de03187f3dff0a271ab-1 {}] regionserver.HStore(327): Store=bb023e4ee9cc2de03187f3dff0a271ab/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:42:01,533 INFO [StoreOpener-bb023e4ee9cc2de03187f3dff0a271ab-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:01,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-09T01:42:01,534 INFO [StoreOpener-bb023e4ee9cc2de03187f3dff0a271ab-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:42:01,534 INFO [StoreOpener-bb023e4ee9cc2de03187f3dff0a271ab-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb023e4ee9cc2de03187f3dff0a271ab columnFamilyName C 2024-12-09T01:42:01,534 DEBUG [StoreOpener-bb023e4ee9cc2de03187f3dff0a271ab-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:01,535 INFO [StoreOpener-bb023e4ee9cc2de03187f3dff0a271ab-1 {}] regionserver.HStore(327): Store=bb023e4ee9cc2de03187f3dff0a271ab/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:42:01,535 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:01,535 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:01,536 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:01,536 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:01,537 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:01,537 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:01,537 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T01:42:01,538 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:01,540 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:42:01,540 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened bb023e4ee9cc2de03187f3dff0a271ab; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73034027, jitterRate=0.08829180896282196}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T01:42:01,540 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:01,541 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for bb023e4ee9cc2de03187f3dff0a271ab: Running coprocessor pre-open hook at 1733708521529Writing region info on filesystem at 1733708521529Initializing all the Stores at 1733708521529Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708521529Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708521530 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708521530Cleaning up temporary data from old regions at 1733708521537 (+7 ms)Running coprocessor post-open hooks at 1733708521540 (+3 ms)Region opened successfully at 1733708521541 (+1 ms) 2024-12-09T01:42:01,542 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., pid=65, masterSystemTime=1733708521525 2024-12-09T01:42:01,543 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:01,543 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:01,544 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=bb023e4ee9cc2de03187f3dff0a271ab, regionState=OPEN, openSeqNum=2, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:01,546 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 because future has completed 2024-12-09T01:42:01,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-12-09T01:42:01,547 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 in 173 msec 2024-12-09T01:42:01,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-12-09T01:42:01,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bb023e4ee9cc2de03187f3dff0a271ab, ASSIGN in 329 msec 2024-12-09T01:42:01,550 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T01:42:01,550 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708521550"}]},"ts":"1733708521550"} 2024-12-09T01:42:01,551 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T01:42:01,552 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T01:42:01,553 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1580 sec 2024-12-09T01:42:02,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-09T01:42:02,544 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:02,545 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@acbcf3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,545 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:02,545 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:02,546 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:02,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:02,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:02,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e9c0b60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:02,547 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:02,548 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:02,548 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:02,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3395fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,549 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:02,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:02,550 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:02,551 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:02,552 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T01:42:02,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is ef6f18c58dc9,37995,1733708453073 2024-12-09T01:42:02,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3331224a 2024-12-09T01:42:02,553 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T01:42:02,554 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40468, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T01:42:02,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a1e55ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,555 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:02,555 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:02,555 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:02,555 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:02,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:02,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b95e807, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:02,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:02,556 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:02,557 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:02,557 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ff2445, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,558 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7642a08c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,558 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:02,558 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:02,559 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:02,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:02,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:02,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b2e17f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:02,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:02,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:02,560 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:02,560 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52e40b13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,561 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ecfb794, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,561 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:02,561 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:02,562 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:02,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:02,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:02,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a3e9200, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:02,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:02,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:02,563 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:02,563 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30bbf801, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,564 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff0f448, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,564 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:02,564 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:02,565 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:02,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:02,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:02,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e9252ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:02,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:02,565 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:02,566 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:02,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@166be6f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,567 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d560643, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,567 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:02,567 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:02,567 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:02,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b69d50c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:02,569 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:02,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f551403, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,570 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aaa706, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,570 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:02,570 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:02,570 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7472fb45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:02,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:02,572 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40584, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:02,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@356c4038, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,573 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@beb69b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,573 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:02,573 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:02,573 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:02,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:02,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:02,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bd4d901, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:02,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:02,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:02,574 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:02,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c804eec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,575 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643da6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,575 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:02,575 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:02,576 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:02,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:02,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:02,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c314ca3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:02,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:02,576 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:02,577 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:02,577 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@228aac10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,579 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4829bcb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,579 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:02,579 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:02,579 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:02,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:02,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:02,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f5a45e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:02,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:02,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:02,580 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:02,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@183bc660, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10d96952, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,581 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:02,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:02,582 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:02,582 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:02,582 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:02,582 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7732c9a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,582 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:02,582 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:02,582 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:02,583 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40652, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:02,583 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4116aeba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:02,584 DEBUG [Thread-1193 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:02,584 DEBUG [Thread-1185 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:02,584 DEBUG [Thread-1187 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:02,584 DEBUG [Thread-1191 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:02,584 DEBUG [Thread-1196 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:02,584 DEBUG [Thread-1189 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:02,585 DEBUG [Thread-1200 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:02,585 DEBUG [Thread-1191 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:02,585 DEBUG [Thread-1204 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:02,585 DEBUG [Thread-1198 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:02,585 DEBUG [Thread-1191 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:02,585 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:02,586 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:02,586 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39832, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:02,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:02,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:02,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:02,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:02,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:02,587 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:42:02,587 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:02,587 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:02,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:02,587 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:02,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:02,588 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:02,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39864, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:02,588 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39852, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:02,588 DEBUG [Thread-1202 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:02,589 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:02,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:02,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:02,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-12-09T01:42:02,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:42:02,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:42:02,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:02,591 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39874, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:02,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:42:02,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:42:02,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:42:02,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T01:42:02,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:42:02,593 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:42:02,594 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:42:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:42:02,595 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:42:02,596 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:42:02,597 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:42:02,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:42:02,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:02,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:42:02,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:02,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:02,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:02,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:02,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:02,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:02,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/8fc706ce5bee4ab1b47e417172cf4d6e is 50, key is test_row_0/A:col10/1733708522601/Put/seqid=0 2024-12-09T01:42:02,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:02,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708582637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:02,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708522631, max timestamp=1733708522642Num uniques 859; 2024-12-09T01:42:02,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708522631, max timestamp=1733708522642Num uniques 859; 2024-12-09T01:42:02,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:42:02,647 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:42:02,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708522631, max timestamp=1733708522642Num uniques 859; 2024-12-09T01:42:02,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:42:02,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742081_1257 (size=14341) 2024-12-09T01:42:02,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:02,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708582674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:02,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:02,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:02,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708582677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:02,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:02,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708582678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:02,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:02,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:02,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:02,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708582681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:02,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:02,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T01:42:02,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T01:42:02,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:02,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:02,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:02,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:02,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:02,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:02,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:02,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708582754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,759 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:02,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:02,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708582783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:02,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:02,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:02,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708582793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:02,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:02,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:02,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708582793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:02,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708582794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:02,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T01:42:02,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:02,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:02,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:02,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:02,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:02,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:02,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T01:42:02,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:02,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708582964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:02,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,966 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:02,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:02,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708582993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:02,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:02,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:03,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:03,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708583004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708583004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:03,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:03,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708583004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:03,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:03,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/8fc706ce5bee4ab1b47e417172cf4d6e 2024-12-09T01:42:03,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T01:42:03,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:03,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:03,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:03,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:03,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:03,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:03,093 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/d66ffb37fe9e49ee9716bb71a1340146 is 50, key is test_row_0/B:col10/1733708522601/Put/seqid=0 2024-12-09T01:42:03,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742082_1258 (size=12001) 2024-12-09T01:42:03,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/d66ffb37fe9e49ee9716bb71a1340146 2024-12-09T01:42:03,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/1322642cca5c4fb990edbafe93b8c265 is 50, key is test_row_0/C:col10/1733708522601/Put/seqid=0 2024-12-09T01:42:03,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742083_1259 (size=12001) 2024-12-09T01:42:03,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/1322642cca5c4fb990edbafe93b8c265 2024-12-09T01:42:03,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/8fc706ce5bee4ab1b47e417172cf4d6e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/8fc706ce5bee4ab1b47e417172cf4d6e 2024-12-09T01:42:03,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/8fc706ce5bee4ab1b47e417172cf4d6e, entries=200, sequenceid=11, filesize=14.0 K 2024-12-09T01:42:03,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/d66ffb37fe9e49ee9716bb71a1340146 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/d66ffb37fe9e49ee9716bb71a1340146 2024-12-09T01:42:03,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/d66ffb37fe9e49ee9716bb71a1340146, entries=150, sequenceid=11, filesize=11.7 K 2024-12-09T01:42:03,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/1322642cca5c4fb990edbafe93b8c265 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/1322642cca5c4fb990edbafe93b8c265 2024-12-09T01:42:03,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/1322642cca5c4fb990edbafe93b8c265, entries=150, sequenceid=11, filesize=11.7 K 2024-12-09T01:42:03,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for bb023e4ee9cc2de03187f3dff0a271ab in 580ms, sequenceid=11, compaction requested=false 2024-12-09T01:42:03,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:03,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-12-09T01:42:03,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:03,219 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-09T01:42:03,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:03,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:03,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:03,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:03,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:03,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:03,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T01:42:03,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/4bb33189a49940039d0ddb5d39ea3f9f is 50, key is test_row_0/A:col10/1733708522675/Put/seqid=0 2024-12-09T01:42:03,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742084_1260 (size=12001) 2024-12-09T01:42:03,249 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/4bb33189a49940039d0ddb5d39ea3f9f 2024-12-09T01:42:03,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/babf64e72a7d491e96566552a55de186 is 50, key is test_row_0/B:col10/1733708522675/Put/seqid=0 2024-12-09T01:42:03,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:03,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:03,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742085_1261 (size=12001) 2024-12-09T01:42:03,299 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/babf64e72a7d491e96566552a55de186 2024-12-09T01:42:03,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/53fec10b353247b099377fa976ccc193 is 50, key is test_row_0/C:col10/1733708522675/Put/seqid=0 2024-12-09T01:42:03,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:03,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708583311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:03,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708583313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:03,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708583313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708583313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:03,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:03,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708583313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:03,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,319 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:03,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:03,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:03,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742086_1262 (size=12001) 2024-12-09T01:42:03,336 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/53fec10b353247b099377fa976ccc193 2024-12-09T01:42:03,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/4bb33189a49940039d0ddb5d39ea3f9f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/4bb33189a49940039d0ddb5d39ea3f9f 2024-12-09T01:42:03,350 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/4bb33189a49940039d0ddb5d39ea3f9f, entries=150, sequenceid=48, filesize=11.7 K 2024-12-09T01:42:03,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/babf64e72a7d491e96566552a55de186 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/babf64e72a7d491e96566552a55de186 2024-12-09T01:42:03,360 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/babf64e72a7d491e96566552a55de186, entries=150, sequenceid=48, filesize=11.7 K 2024-12-09T01:42:03,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/53fec10b353247b099377fa976ccc193 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/53fec10b353247b099377fa976ccc193 2024-12-09T01:42:03,373 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/53fec10b353247b099377fa976ccc193, entries=150, sequenceid=48, filesize=11.7 K 2024-12-09T01:42:03,374 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for bb023e4ee9cc2de03187f3dff0a271ab in 156ms, sequenceid=48, compaction requested=false 2024-12-09T01:42:03,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:03,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:03,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-12-09T01:42:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-12-09T01:42:03,387 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-12-09T01:42:03,387 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 784 msec 2024-12-09T01:42:03,389 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 801 msec 2024-12-09T01:42:03,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T01:42:03,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:03,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:03,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:03,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:03,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:03,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:03,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:03,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/3ffff014b671426b8902cd650015b0ad is 50, key is test_row_0/A:col10/1733708523284/Put/seqid=0 2024-12-09T01:42:03,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742087_1263 (size=14341) 2024-12-09T01:42:03,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/3ffff014b671426b8902cd650015b0ad 2024-12-09T01:42:03,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/e38dc64033e447e8a2f8ad955697f47e is 50, key is test_row_0/B:col10/1733708523284/Put/seqid=0 2024-12-09T01:42:03,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708583528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:03,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:03,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708583529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:03,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:03,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,537 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:03,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742088_1264 (size=12001) 2024-12-09T01:42:03,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/e38dc64033e447e8a2f8ad955697f47e 2024-12-09T01:42:03,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/cd3bdf81fb14484b903856be0c7cfb14 is 50, key is test_row_0/C:col10/1733708523284/Put/seqid=0 2024-12-09T01:42:03,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742089_1265 (size=12001) 2024-12-09T01:42:03,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/cd3bdf81fb14484b903856be0c7cfb14 2024-12-09T01:42:03,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/3ffff014b671426b8902cd650015b0ad as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/3ffff014b671426b8902cd650015b0ad 2024-12-09T01:42:03,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/3ffff014b671426b8902cd650015b0ad, entries=200, sequenceid=60, filesize=14.0 K 2024-12-09T01:42:03,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/e38dc64033e447e8a2f8ad955697f47e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/e38dc64033e447e8a2f8ad955697f47e 2024-12-09T01:42:03,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/e38dc64033e447e8a2f8ad955697f47e, entries=150, sequenceid=60, filesize=11.7 K 2024-12-09T01:42:03,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/cd3bdf81fb14484b903856be0c7cfb14 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/cd3bdf81fb14484b903856be0c7cfb14 2024-12-09T01:42:03,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/cd3bdf81fb14484b903856be0c7cfb14, entries=150, sequenceid=60, filesize=11.7 K 2024-12-09T01:42:03,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for bb023e4ee9cc2de03187f3dff0a271ab in 182ms, sequenceid=60, compaction requested=true 2024-12-09T01:42:03,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:03,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:42:03,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:03,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:42:03,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:03,609 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:03,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:42:03,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:03,609 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:03,611 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:03,611 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/B is initiating minor compaction (all files) 2024-12-09T01:42:03,611 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/B in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:03,611 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/d66ffb37fe9e49ee9716bb71a1340146, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/babf64e72a7d491e96566552a55de186, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/e38dc64033e447e8a2f8ad955697f47e] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=35.2 K 2024-12-09T01:42:03,611 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:03,612 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/A is initiating minor compaction (all files) 2024-12-09T01:42:03,612 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/A in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:03,612 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/8fc706ce5bee4ab1b47e417172cf4d6e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/4bb33189a49940039d0ddb5d39ea3f9f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/3ffff014b671426b8902cd650015b0ad] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=39.7 K 2024-12-09T01:42:03,612 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting d66ffb37fe9e49ee9716bb71a1340146, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733708522594 2024-12-09T01:42:03,613 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fc706ce5bee4ab1b47e417172cf4d6e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733708522594 2024-12-09T01:42:03,613 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting babf64e72a7d491e96566552a55de186, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733708522675 2024-12-09T01:42:03,614 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bb33189a49940039d0ddb5d39ea3f9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733708522675 2024-12-09T01:42:03,614 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting e38dc64033e447e8a2f8ad955697f47e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733708523284 2024-12-09T01:42:03,614 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ffff014b671426b8902cd650015b0ad, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733708523284 2024-12-09T01:42:03,639 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#A#compaction#222 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:03,640 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/361bed845bcd4a669f5dc5134ee4b3aa is 50, key is test_row_0/A:col10/1733708523284/Put/seqid=0 2024-12-09T01:42:03,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:03,649 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#B#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:03,650 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/eb59cf4ef2254848ae82e653ad37c53c is 50, key is test_row_0/B:col10/1733708523284/Put/seqid=0 2024-12-09T01:42:03,650 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733708523516, max timestamp=1733708523644Num uniques 859; 2024-12-09T01:42:03,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T01:42:03,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733708523516, max timestamp=1733708523644Num uniques 859; 2024-12-09T01:42:03,653 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-12-09T01:42:03,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=57.35 KB heapSize=162.34 KB 2024-12-09T01:42:03,654 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708523516, max timestamp=1733708523644Num uniques 859; 2024-12-09T01:42:03,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:42:03,655 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:03,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:03,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:03,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:03,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:03,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:03,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/a83ff8724a2946268a99205e4a94cbe6 is 50, key is test_row_0/A:col10/1733708523518/Put/seqid=0 2024-12-09T01:42:03,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742091_1267 (size=12104) 2024-12-09T01:42:03,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742090_1266 (size=12104) 2024-12-09T01:42:03,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742092_1268 (size=14341) 2024-12-09T01:42:03,689 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/eb59cf4ef2254848ae82e653ad37c53c as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/eb59cf4ef2254848ae82e653ad37c53c 2024-12-09T01:42:03,696 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/B of bb023e4ee9cc2de03187f3dff0a271ab into eb59cf4ef2254848ae82e653ad37c53c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:03,696 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:03,696 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/B, priority=13, startTime=1733708523609; duration=0sec 2024-12-09T01:42:03,696 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:03,696 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:B 2024-12-09T01:42:03,696 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:03,697 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:03,697 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/C is initiating minor compaction (all files) 2024-12-09T01:42:03,698 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/C in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:03,698 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/1322642cca5c4fb990edbafe93b8c265, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/53fec10b353247b099377fa976ccc193, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/cd3bdf81fb14484b903856be0c7cfb14] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=35.2 K 2024-12-09T01:42:03,698 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 1322642cca5c4fb990edbafe93b8c265, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733708522594 2024-12-09T01:42:03,698 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 53fec10b353247b099377fa976ccc193, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733708522675 2024-12-09T01:42:03,699 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting cd3bdf81fb14484b903856be0c7cfb14, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733708523284 2024-12-09T01:42:03,721 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#C#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:03,721 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/21550ed2404d4e76949853c707df51cc is 50, key is test_row_0/C:col10/1733708523284/Put/seqid=0 2024-12-09T01:42:03,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-12-09T01:42:03,734 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:03,735 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:42:03,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-12-09T01:42:03,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T01:42:03,737 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:42:03,738 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:42:03,738 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:42:03,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708523716, max timestamp=1733708523741Num uniques 859; 2024-12-09T01:42:03,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708523716, max timestamp=1733708523741Num uniques 859; 2024-12-09T01:42:03,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:42:03,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:42:03,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708523716, max timestamp=1733708523741Num uniques 859; 2024-12-09T01:42:03,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:42:03,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742093_1269 (size=12104) 2024-12-09T01:42:03,758 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/21550ed2404d4e76949853c707df51cc as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/21550ed2404d4e76949853c707df51cc 2024-12-09T01:42:03,764 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/C of bb023e4ee9cc2de03187f3dff0a271ab into 21550ed2404d4e76949853c707df51cc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:03,764 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:03,764 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/C, priority=13, startTime=1733708523609; duration=0sec 2024-12-09T01:42:03,764 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:03,764 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:C 2024-12-09T01:42:03,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733708523751, max timestamp=1733708523825Num uniques 1068; 2024-12-09T01:42:03,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-09T01:42:03,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733708523751, max timestamp=1733708523825Num uniques 1068; 2024-12-09T01:42:03,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-09T01:42:03,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1733708523751, max timestamp=1733708523825Num uniques 1068; 2024-12-09T01:42:03,829 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-12-09T01:42:03,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T01:42:03,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733708523848, max timestamp=1733708523856Num uniques 1027; 2024-12-09T01:42:03,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-09T01:42:03,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733708523848, max timestamp=1733708523856Num uniques 1027; 2024-12-09T01:42:03,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-09T01:42:03,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1733708523848, max timestamp=1733708523856Num uniques 1027; 2024-12-09T01:42:03,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-12-09T01:42:03,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T01:42:03,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:03,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:03,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:03,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:03,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:03,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:03,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:03,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708583887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:03,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:03,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708583889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:03,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733708523881, max timestamp=1733708523896Num uniques 1086; 2024-12-09T01:42:03,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-09T01:42:03,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733708523881, max timestamp=1733708523896Num uniques 1086; 2024-12-09T01:42:03,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:03,901 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-09T01:42:03,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1733708523881, max timestamp=1733708523898Num uniques 1086; 2024-12-09T01:42:03,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-12-09T01:42:03,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:03,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:03,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733708523971, max timestamp=1733708523987Num uniques 1095; 2024-12-09T01:42:03,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-12-09T01:42:03,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733708523971, max timestamp=1733708523987Num uniques 1095; 2024-12-09T01:42:03,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733708523971, max timestamp=1733708523987Num uniques 1095; 2024-12-09T01:42:03,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-12-09T01:42:03,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-12-09T01:42:04,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733708524000, max timestamp=1733708524033Num uniques 1054; 2024-12-09T01:42:04,037 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-12-09T01:42:04,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1733708524000, max timestamp=1733708524033Num uniques 1054; 2024-12-09T01:42:04,040 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=9340, suffix heap size=137584,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=204 2024-12-09T01:42:04,042 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733708524024, max timestamp=1733708524037Num uniques 1054; 2024-12-09T01:42:04,043 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-09T01:42:04,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T01:42:04,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:04,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:04,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:04,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:04,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:04,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:04,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T01:42:04,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:04,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708584067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:04,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:04,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708584069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:04,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733708524053, max timestamp=1733708524072Num uniques 1113; 2024-12-09T01:42:04,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-12-09T01:42:04,077 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733708524053, max timestamp=1733708524072Num uniques 1063; 2024-12-09T01:42:04,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-12-09T01:42:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:04,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733708524053, max timestamp=1733708524072Num uniques 1013; 2024-12-09T01:42:04,083 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=6870, suffix heap size=133320,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=150 2024-12-09T01:42:04,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/a83ff8724a2946268a99205e4a94cbe6 2024-12-09T01:42:04,098 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/361bed845bcd4a669f5dc5134ee4b3aa as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/361bed845bcd4a669f5dc5134ee4b3aa 2024-12-09T01:42:04,103 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/A of bb023e4ee9cc2de03187f3dff0a271ab into 361bed845bcd4a669f5dc5134ee4b3aa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:04,103 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:04,103 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/A, priority=13, startTime=1733708523609; duration=0sec 2024-12-09T01:42:04,104 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:04,104 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:A 2024-12-09T01:42:04,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/df010469cd09485b9d3be0ad8363b944 is 50, key is test_row_0/B:col10/1733708523518/Put/seqid=0 2024-12-09T01:42:04,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742094_1270 (size=12001) 2024-12-09T01:42:04,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.79 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/df010469cd09485b9d3be0ad8363b944 2024-12-09T01:42:04,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/1d4878776e784b94931c6febeca6d510 is 50, key is test_row_0/C:col10/1733708523518/Put/seqid=0 2024-12-09T01:42:04,137 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733708524120, max timestamp=1733708524131Num uniques 1072; 2024-12-09T01:42:04,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-12-09T01:42:04,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733708524120, max timestamp=1733708524131Num uniques 1009; 2024-12-09T01:42:04,140 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=7872, suffix heap size=132904,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=172 2024-12-09T01:42:04,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733708524120, max timestamp=1733708524131Num uniques 1072; 2024-12-09T01:42:04,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-12-09T01:42:04,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742095_1271 (size=12001) 2024-12-09T01:42:04,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733708524170, max timestamp=1733708524186Num uniques 1031; 2024-12-09T01:42:04,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-12-09T01:42:04,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733708524170, max timestamp=1733708524186Num uniques 1031; 2024-12-09T01:42:04,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=231, cellSize=10576, totalHeapSize=24264, min timestamp=1733708524170, max timestamp=1733708524186Num uniques 1031; 2024-12-09T01:42:04,191 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-12-09T01:42:04,192 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=10576, suffix heap size=135192,new segment heap size=24264 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=231 2024-12-09T01:42:04,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T01:42:04,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:04,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:04,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:04,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:04,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:04,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:04,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733708524186, max timestamp=1733708524220Num uniques 1090; 2024-12-09T01:42:04,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-12-09T01:42:04,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733708524186, max timestamp=1733708524220Num uniques 1090; 2024-12-09T01:42:04,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-12-09T01:42:04,225 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1733708524186, max timestamp=1733708524220Num uniques 1090; 2024-12-09T01:42:04,227 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49920, new segment data size=10990, suffix heap size=141328,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1090, new segment cells count=240 2024-12-09T01:42:04,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:04,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708584258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:04,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708584257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:04,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:04,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9104, totalHeapSize=20936, min timestamp=1733708524251, max timestamp=1733708524263Num uniques 1099; 2024-12-09T01:42:04,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9104, totalHeapSize=20936, min timestamp=1733708524251, max timestamp=1733708524263Num uniques 1099; 2024-12-09T01:42:04,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9104, totalHeapSize=20936, min timestamp=1733708524251, max timestamp=1733708524263Num uniques 1099; 2024-12-09T01:42:04,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50324, new segment data size=9104, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-12-09T01:42:04,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50324, new segment data size=9104, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-12-09T01:42:04,270 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50324, new segment data size=9104, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-12-09T01:42:04,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:04,318 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733708524298, max timestamp=1733708524314Num uniques 1058; 2024-12-09T01:42:04,319 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=9520, suffix heap size=138000,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-12-09T01:42:04,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733708524298, max timestamp=1733708524314Num uniques 1058; 2024-12-09T01:42:04,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=9520, suffix heap size=138000,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-12-09T01:42:04,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1733708524298, max timestamp=1733708524314Num uniques 1058; 2024-12-09T01:42:04,323 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=9520, suffix heap size=138000,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-12-09T01:42:04,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T01:42:04,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:04,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:04,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:04,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:04,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:04,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:04,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T01:42:04,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733708524353, max timestamp=1733708524364Num uniques 1067; 2024-12-09T01:42:04,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733708524353, max timestamp=1733708524364Num uniques 1067; 2024-12-09T01:42:04,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-12-09T01:42:04,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-12-09T01:42:04,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1733708524353, max timestamp=1733708524364Num uniques 1067; 2024-12-09T01:42:04,370 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-12-09T01:42:04,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733708524385, max timestamp=1733708524398Num uniques 1026; 2024-12-09T01:42:04,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733708524385, max timestamp=1733708524398Num uniques 1026; 2024-12-09T01:42:04,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1733708524385, max timestamp=1733708524398Num uniques 1026; 2024-12-09T01:42:04,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-12-09T01:42:04,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-12-09T01:42:04,403 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=10346, suffix heap size=134672,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=226 2024-12-09T01:42:04,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=335, cellSize=15340, totalHeapSize=35080, min timestamp=1733708524413, max timestamp=1733708524432Num uniques 1085; 2024-12-09T01:42:04,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=15340, suffix heap size=140808,new segment heap size=35080 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=335 2024-12-09T01:42:04,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=335, cellSize=15340, totalHeapSize=35080, min timestamp=1733708524413, max timestamp=1733708524432Num uniques 1085; 2024-12-09T01:42:04,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=335, cellSize=15340, totalHeapSize=35080, min timestamp=1733708524413, max timestamp=1733708524432Num uniques 1085; 2024-12-09T01:42:04,437 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=15340, suffix heap size=140808,new segment heap size=35080 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=335 2024-12-09T01:42:04,438 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49690, new segment data size=15340, suffix heap size=140808,new segment heap size=35080 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1085, new segment cells count=335 2024-12-09T01:42:04,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:04,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708584459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:04,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:04,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708584463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:04,470 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=344, cellSize=15754, totalHeapSize=36016, min timestamp=1733708524450, max timestamp=1733708524465Num uniques 1194; 2024-12-09T01:42:04,471 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54684, new segment data size=15754, suffix heap size=152144,new segment heap size=36016 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1194, new segment cells count=344 2024-12-09T01:42:04,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733708524450, max timestamp=1733708524465Num uniques 1194; 2024-12-09T01:42:04,472 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=54684, new segment data size=6870, suffix heap size=152144,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1194, new segment cells count=150 2024-12-09T01:42:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:04,475 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=344, cellSize=15754, totalHeapSize=36016, min timestamp=1733708524450, max timestamp=1733708524465Num uniques 1194; 2024-12-09T01:42:04,475 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=54684, new segment data size=15754, suffix heap size=152144,new segment heap size=36016 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1194, new segment cells count=344 2024-12-09T01:42:04,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,476 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:04,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=253, cellSize=11585, totalHeapSize=26552, min timestamp=1733708524501, max timestamp=1733708524510Num uniques 1009; 2024-12-09T01:42:04,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46211, new segment data size=11585, suffix heap size=132904,new segment heap size=26552 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=253 2024-12-09T01:42:04,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=253, cellSize=11585, totalHeapSize=26552, min timestamp=1733708524501, max timestamp=1733708524510Num uniques 1203; 2024-12-09T01:42:04,516 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=55095, new segment data size=11585, suffix heap size=153080,new segment heap size=26552 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1203, new segment cells count=253 2024-12-09T01:42:04,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T01:42:04,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:04,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:04,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:04,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:04,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:04,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:04,515 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=253, cellSize=11585, totalHeapSize=26552, min timestamp=1733708524501, max timestamp=1733708524510Num uniques 1203; 2024-12-09T01:42:04,519 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=55095, new segment data size=11585, suffix heap size=153080,new segment heap size=26552 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1203, new segment cells count=253 2024-12-09T01:42:04,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.79 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/1d4878776e784b94931c6febeca6d510 2024-12-09T01:42:04,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/a83ff8724a2946268a99205e4a94cbe6 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/a83ff8724a2946268a99205e4a94cbe6 2024-12-09T01:42:04,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/a83ff8724a2946268a99205e4a94cbe6, entries=200, sequenceid=82, filesize=14.0 K 2024-12-09T01:42:04,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/df010469cd09485b9d3be0ad8363b944 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/df010469cd09485b9d3be0ad8363b944 2024-12-09T01:42:04,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1733708524532, max timestamp=1733708524566Num uniques 1112; 2024-12-09T01:42:04,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50922, new segment data size=7412, suffix heap size=143616,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1112, new segment cells count=162 2024-12-09T01:42:04,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1733708524532, max timestamp=1733708524566Num uniques 1112; 2024-12-09T01:42:04,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50922, new segment data size=7412, suffix heap size=143616,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1112, new segment cells count=162 2024-12-09T01:42:04,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1733708524532, max timestamp=1733708524566Num uniques 1112; 2024-12-09T01:42:04,572 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50922, new segment data size=7412, suffix heap size=143616,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1112, new segment cells count=162 2024-12-09T01:42:04,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/df010469cd09485b9d3be0ad8363b944, entries=150, sequenceid=82, filesize=11.7 K 2024-12-09T01:42:04,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/1d4878776e784b94931c6febeca6d510 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/1d4878776e784b94931c6febeca6d510 2024-12-09T01:42:04,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/1d4878776e784b94931c6febeca6d510, entries=150, sequenceid=82, filesize=11.7 K 2024-12-09T01:42:04,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~32.75 KB/33540, heapSize ~84.28 KB/86304, currentSize=53.67 KB/54960 for bb023e4ee9cc2de03187f3dff0a271ab in 928ms, sequenceid=82, compaction requested=false 2024-12-09T01:42:04,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:04,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:04,584 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=67.09 KB heapSize=185.48 KB 2024-12-09T01:42:04,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:04,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:04,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:04,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:04,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:04,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:04,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/afd0a777705d424da66135e486c6b79a is 50, key is test_row_0/A:col10/1733708524581/Put/seqid=0 2024-12-09T01:42:04,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742096_1272 (size=12301) 2024-12-09T01:42:04,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/afd0a777705d424da66135e486c6b79a 2024-12-09T01:42:04,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:04,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708584614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:04,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:04,624 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/eeb673e1bfe44a71839d8a0fd2c7c299 is 50, key is test_row_0/B:col10/1733708524581/Put/seqid=0 2024-12-09T01:42:04,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:04,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708584616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:04,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:04,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708584617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:04,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,626 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:04,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:04,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708584618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:04,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:04,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708584619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:04,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742097_1273 (size=12301) 2024-12-09T01:42:04,644 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/eeb673e1bfe44a71839d8a0fd2c7c299 2024-12-09T01:42:04,658 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/69319c29107645b5ab58dc50529dbcc1 is 50, key is test_row_0/C:col10/1733708524581/Put/seqid=0 2024-12-09T01:42:04,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742098_1274 (size=12301) 2024-12-09T01:42:04,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/69319c29107645b5ab58dc50529dbcc1 2024-12-09T01:42:04,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T01:42:04,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:04,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:04,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:04,672 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:04,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:04,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/afd0a777705d424da66135e486c6b79a as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/afd0a777705d424da66135e486c6b79a 2024-12-09T01:42:04,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:04,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/afd0a777705d424da66135e486c6b79a, entries=150, sequenceid=405, filesize=12.0 K 2024-12-09T01:42:04,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/eeb673e1bfe44a71839d8a0fd2c7c299 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/eeb673e1bfe44a71839d8a0fd2c7c299 2024-12-09T01:42:04,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/eeb673e1bfe44a71839d8a0fd2c7c299, entries=150, sequenceid=405, filesize=12.0 K 2024-12-09T01:42:04,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/69319c29107645b5ab58dc50529dbcc1 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/69319c29107645b5ab58dc50529dbcc1 2024-12-09T01:42:04,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/69319c29107645b5ab58dc50529dbcc1, entries=150, sequenceid=405, filesize=12.0 K 2024-12-09T01:42:04,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~205.36 KB/210288, currentSize=107.34 KB/109920 for bb023e4ee9cc2de03187f3dff0a271ab in 112ms, sequenceid=405, compaction requested=true 2024-12-09T01:42:04,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:04,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:42:04,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:04,695 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:04,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:42:04,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:04,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:42:04,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:04,695 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:04,696 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38746 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:04,696 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/A is initiating minor compaction (all files) 2024-12-09T01:42:04,697 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/A in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:04,697 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:04,697 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/B is initiating minor compaction (all files) 2024-12-09T01:42:04,697 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/361bed845bcd4a669f5dc5134ee4b3aa, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/a83ff8724a2946268a99205e4a94cbe6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/afd0a777705d424da66135e486c6b79a] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=37.8 K 2024-12-09T01:42:04,697 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/B in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:04,697 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/eb59cf4ef2254848ae82e653ad37c53c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/df010469cd09485b9d3be0ad8363b944, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/eeb673e1bfe44a71839d8a0fd2c7c299] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=35.6 K 2024-12-09T01:42:04,698 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 361bed845bcd4a669f5dc5134ee4b3aa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733708523284 2024-12-09T01:42:04,698 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting a83ff8724a2946268a99205e4a94cbe6, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733708523516 2024-12-09T01:42:04,698 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting eb59cf4ef2254848ae82e653ad37c53c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733708523284 2024-12-09T01:42:04,699 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting df010469cd09485b9d3be0ad8363b944, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733708523518 2024-12-09T01:42:04,699 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting afd0a777705d424da66135e486c6b79a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733708524576 2024-12-09T01:42:04,699 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting eeb673e1bfe44a71839d8a0fd2c7c299, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733708524576 2024-12-09T01:42:04,712 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#A#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:04,712 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/babd66918b8e42ce97f417d472216c20 is 50, key is test_row_0/A:col10/1733708524581/Put/seqid=0 2024-12-09T01:42:04,722 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#B#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:04,722 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/fbfd255687f846b582208f65bd75cd43 is 50, key is test_row_0/B:col10/1733708524581/Put/seqid=0 2024-12-09T01:42:04,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:04,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T01:42:04,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708524616, max timestamp=1733708524737Num uniques 859; 2024-12-09T01:42:04,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:42:04,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708524616, max timestamp=1733708524737Num uniques 859; 2024-12-09T01:42:04,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:42:04,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708524616, max timestamp=1733708524737Num uniques 859; 2024-12-09T01:42:04,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:42:04,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:04,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:04,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:04,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:04,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:04,741 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:04,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742099_1275 (size=12507) 2024-12-09T01:42:04,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742100_1276 (size=12507) 2024-12-09T01:42:04,751 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/babd66918b8e42ce97f417d472216c20 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/babd66918b8e42ce97f417d472216c20 2024-12-09T01:42:04,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/b1982169ca5e417b9334456d15e41b3b is 50, key is test_row_0/A:col10/1733708524617/Put/seqid=0 2024-12-09T01:42:04,758 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/fbfd255687f846b582208f65bd75cd43 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/fbfd255687f846b582208f65bd75cd43 2024-12-09T01:42:04,763 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/A of bb023e4ee9cc2de03187f3dff0a271ab into babd66918b8e42ce97f417d472216c20(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:04,763 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:04,763 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/A, priority=13, startTime=1733708524695; duration=0sec 2024-12-09T01:42:04,763 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:04,763 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:A 2024-12-09T01:42:04,764 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:04,766 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36406 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:04,766 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/C is initiating minor compaction (all files) 2024-12-09T01:42:04,766 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/C in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:04,766 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/21550ed2404d4e76949853c707df51cc, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/1d4878776e784b94931c6febeca6d510, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/69319c29107645b5ab58dc50529dbcc1] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=35.6 K 2024-12-09T01:42:04,767 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21550ed2404d4e76949853c707df51cc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1733708523284 2024-12-09T01:42:04,768 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d4878776e784b94931c6febeca6d510, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1733708523518 2024-12-09T01:42:04,768 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69319c29107645b5ab58dc50529dbcc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733708524576 2024-12-09T01:42:04,769 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/B of bb023e4ee9cc2de03187f3dff0a271ab into fbfd255687f846b582208f65bd75cd43(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:04,769 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:04,769 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/B, priority=13, startTime=1733708524695; duration=0sec 2024-12-09T01:42:04,769 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:04,769 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:B 2024-12-09T01:42:04,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742101_1277 (size=12301) 2024-12-09T01:42:04,781 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#C#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:04,782 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/67afcbf187cc46f483590567fb671899 is 50, key is test_row_0/C:col10/1733708524581/Put/seqid=0 2024-12-09T01:42:04,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742102_1278 (size=12507) 2024-12-09T01:42:04,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708524758, max timestamp=1733708524786Num uniques 859; 2024-12-09T01:42:04,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708524758, max timestamp=1733708524786Num uniques 859; 2024-12-09T01:42:04,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708524758, max timestamp=1733708524786Num uniques 859; 2024-12-09T01:42:04,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:42:04,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:42:04,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:42:04,799 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/67afcbf187cc46f483590567fb671899 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/67afcbf187cc46f483590567fb671899 2024-12-09T01:42:04,806 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/C of bb023e4ee9cc2de03187f3dff0a271ab into 67afcbf187cc46f483590567fb671899(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:04,806 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:04,806 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/C, priority=13, startTime=1733708524695; duration=0sec 2024-12-09T01:42:04,806 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:04,806 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:C 2024-12-09T01:42:04,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:04,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708584817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:04,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T01:42:04,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:04,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:04,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:04,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:04,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:04,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:04,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:04,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708584819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:04,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733708524808, max timestamp=1733708524822Num uniques 1118; 2024-12-09T01:42:04,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-12-09T01:42:04,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:04,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:04,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733708524815, max timestamp=1733708524822Num uniques 1118; 2024-12-09T01:42:04,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=6870, suffix heap size=144240,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=150 2024-12-09T01:42:04,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1733708524815, max timestamp=1733708524822Num uniques 1118; 2024-12-09T01:42:04,832 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=6870, suffix heap size=144240,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=150 2024-12-09T01:42:04,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733708524834, max timestamp=1733708524872Num uniques 1077; 2024-12-09T01:42:04,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-09T01:42:04,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733708524834, max timestamp=1733708524872Num uniques 1009; 2024-12-09T01:42:04,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=12682, suffix heap size=132904,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=277 2024-12-09T01:42:04,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1733708524863, max timestamp=1733708524872Num uniques 1009; 2024-12-09T01:42:04,877 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8102, suffix heap size=132904,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=177 2024-12-09T01:42:04,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T01:42:04,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733708524894, max timestamp=1733708524914Num uniques 1136; 2024-12-09T01:42:04,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-09T01:42:04,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733708524894, max timestamp=1733708524914Num uniques 1136; 2024-12-09T01:42:04,919 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-09T01:42:04,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733708524894, max timestamp=1733708524914Num uniques 1036; 2024-12-09T01:42:04,920 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-12-09T01:42:04,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733708524934, max timestamp=1733708524948Num uniques 1045; 2024-12-09T01:42:04,953 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-12-09T01:42:04,954 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=295, cellSize=13510, totalHeapSize=30920, min timestamp=1733708524934, max timestamp=1733708524948Num uniques 1045; 2024-12-09T01:42:04,954 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=13510, suffix heap size=136648,new segment heap size=30920 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=295 2024-12-09T01:42:04,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733708524934, max timestamp=1733708524948Num uniques 1045; 2024-12-09T01:42:04,957 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-09T01:42:04,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T01:42:04,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:04,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:04,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:04,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:04,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:04,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:04,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733708524973, max timestamp=1733708524982Num uniques 1154; 2024-12-09T01:42:04,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-12-09T01:42:04,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733708524973, max timestamp=1733708524982Num uniques 1154; 2024-12-09T01:42:04,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52850, new segment data size=7050, suffix heap size=147984,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1154, new segment cells count=154 2024-12-09T01:42:04,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733708524973, max timestamp=1733708524982Num uniques 1054; 2024-12-09T01:42:04,988 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-09T01:42:05,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733708525003, max timestamp=1733708525015Num uniques 1013; 2024-12-09T01:42:05,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733708525003, max timestamp=1733708525015Num uniques 1013; 2024-12-09T01:42:05,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=12038, suffix heap size=133320,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=263 2024-12-09T01:42:05,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=12038, suffix heap size=133320,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=263 2024-12-09T01:42:05,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=263, cellSize=12038, totalHeapSize=27592, min timestamp=1733708525003, max timestamp=1733708525015Num uniques 1013; 2024-12-09T01:42:05,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=12038, suffix heap size=133320,new segment heap size=27592 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=263 2024-12-09T01:42:05,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733708525052, max timestamp=1733708525057Num uniques 1122; 2024-12-09T01:42:05,063 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-12-09T01:42:05,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733708525052, max timestamp=1733708525057Num uniques 1122; 2024-12-09T01:42:05,064 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-12-09T01:42:05,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733708525052, max timestamp=1733708525057Num uniques 1122; 2024-12-09T01:42:05,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51382, new segment data size=7872, suffix heap size=144656,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1122, new segment cells count=172 2024-12-09T01:42:05,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733708525083, max timestamp=1733708525105Num uniques 1031; 2024-12-09T01:42:05,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-09T01:42:05,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733708525083, max timestamp=1733708525105Num uniques 1031; 2024-12-09T01:42:05,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-09T01:42:05,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1733708525083, max timestamp=1733708525105Num uniques 1031; 2024-12-09T01:42:05,111 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-12-09T01:42:05,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T01:42:05,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:05,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:05,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:05,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:05,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:05,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:05,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733708525128, max timestamp=1733708525133Num uniques 1040; 2024-12-09T01:42:05,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-09T01:42:05,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733708525128, max timestamp=1733708525133Num uniques 1040; 2024-12-09T01:42:05,138 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-09T01:42:05,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1733708525128, max timestamp=1733708525133Num uniques 1040; 2024-12-09T01:42:05,141 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-12-09T01:42:05,174 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733708525152, max timestamp=1733708525169Num uniques 1049; 2024-12-09T01:42:05,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-09T01:42:05,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733708525152, max timestamp=1733708525169Num uniques 1049; 2024-12-09T01:42:05,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-09T01:42:05,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1733708525152, max timestamp=1733708525169Num uniques 1049; 2024-12-09T01:42:05,175 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-12-09T01:42:05,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/b1982169ca5e417b9334456d15e41b3b 2024-12-09T01:42:05,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/046557ea4e9e46e5a84856c009d151ea is 50, key is test_row_0/B:col10/1733708524617/Put/seqid=0 2024-12-09T01:42:05,207 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733708525193, max timestamp=1733708525203Num uniques 1058; 2024-12-09T01:42:05,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-12-09T01:42:05,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733708525193, max timestamp=1733708525203Num uniques 1058; 2024-12-09T01:42:05,208 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-12-09T01:42:05,210 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=158, cellSize=7230, totalHeapSize=16672, min timestamp=1733708525193, max timestamp=1733708525203Num uniques 1058; 2024-12-09T01:42:05,210 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=7230, suffix heap size=138000,new segment heap size=16672 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=158 2024-12-09T01:42:05,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742103_1279 (size=12301) 2024-12-09T01:42:05,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/046557ea4e9e46e5a84856c009d151ea 2024-12-09T01:42:05,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/c09c0e086ff14789a320ec66acd4be27 is 50, key is test_row_0/C:col10/1733708524617/Put/seqid=0 2024-12-09T01:42:05,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742104_1280 (size=12301) 2024-12-09T01:42:05,233 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/c09c0e086ff14789a320ec66acd4be27 2024-12-09T01:42:05,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/b1982169ca5e417b9334456d15e41b3b as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/b1982169ca5e417b9334456d15e41b3b 2024-12-09T01:42:05,245 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/b1982169ca5e417b9334456d15e41b3b, entries=150, sequenceid=426, filesize=12.0 K 2024-12-09T01:42:05,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/046557ea4e9e46e5a84856c009d151ea as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/046557ea4e9e46e5a84856c009d151ea 2024-12-09T01:42:05,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733708525229, max timestamp=1733708525244Num uniques 1017; 2024-12-09T01:42:05,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-12-09T01:42:05,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733708525229, max timestamp=1733708525244Num uniques 1017; 2024-12-09T01:42:05,259 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-12-09T01:42:05,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1733708525229, max timestamp=1733708525244Num uniques 1017; 2024-12-09T01:42:05,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46572, new segment data size=9932, suffix heap size=133736,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1017, new segment cells count=217 2024-12-09T01:42:05,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/046557ea4e9e46e5a84856c009d151ea, entries=150, sequenceid=426, filesize=12.0 K 2024-12-09T01:42:05,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/c09c0e086ff14789a320ec66acd4be27 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/c09c0e086ff14789a320ec66acd4be27 2024-12-09T01:42:05,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/c09c0e086ff14789a320ec66acd4be27, entries=150, sequenceid=426, filesize=12.0 K 2024-12-09T01:42:05,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=93.93 KB/96180 for bb023e4ee9cc2de03187f3dff0a271ab in 537ms, sequenceid=426, compaction requested=false 2024-12-09T01:42:05,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:05,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=100.63 KB heapSize=279.94 KB 2024-12-09T01:42:05,275 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:05,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:05,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:05,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:05,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:05,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:05,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/9fb5e03323964c7c847627ac55e099c1 is 50, key is test_row_0/A:col10/1733708525258/Put/seqid=0 2024-12-09T01:42:05,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T01:42:05,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:05,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:05,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:05,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:05,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:05,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:05,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:05,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708585296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:05,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,306 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742105_1281 (size=17181) 2024-12-09T01:42:05,307 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:05,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:05,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708585300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:05,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,307 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:05,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708585304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:05,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708585306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:05,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708585306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,315 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:05,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:05,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708585413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:05,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:05,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708585413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:05,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,416 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:05,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,417 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:05,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708585423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:05,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:05,428 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708585424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:05,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,429 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:05,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:05,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708585424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:05,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:05,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:05,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T01:42:05,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:05,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:05,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:05,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:05,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:05,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:05,564 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T01:42:05,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T01:42:05,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:05,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:05,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:05,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:05,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:05,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:05,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:05,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708585623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:05,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708585623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:05,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708585634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:05,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:05,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708585634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:05,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:05,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:05,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708585634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:05,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=666 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/9fb5e03323964c7c847627ac55e099c1 2024-12-09T01:42:05,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/ee79d1767de840f6a70c1919736cd6df is 50, key is test_row_0/B:col10/1733708525258/Put/seqid=0 2024-12-09T01:42:05,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742106_1282 (size=12301) 2024-12-09T01:42:05,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=666 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/ee79d1767de840f6a70c1919736cd6df 2024-12-09T01:42:05,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/0b5250342e9842d682d5808329202c7f is 50, key is test_row_0/C:col10/1733708525258/Put/seqid=0 2024-12-09T01:42:05,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T01:42:05,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:05,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:05,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:05,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:05,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:05,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:05,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742107_1283 (size=12301) 2024-12-09T01:42:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T01:42:05,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T01:42:05,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:05,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:05,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:05,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:05,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:05,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:05,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708585933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:05,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708585933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:05,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,937 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:05,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,938 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:05,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708585943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:05,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:05,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708585944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:05,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:05,948 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:05,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708585944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:05,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:05,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:06,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T01:42:06,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:06,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:06,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:06,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:06,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:06,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:06,152 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=666 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/0b5250342e9842d682d5808329202c7f 2024-12-09T01:42:06,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/9fb5e03323964c7c847627ac55e099c1 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/9fb5e03323964c7c847627ac55e099c1 2024-12-09T01:42:06,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/9fb5e03323964c7c847627ac55e099c1, entries=250, sequenceid=666, filesize=16.8 K 2024-12-09T01:42:06,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/ee79d1767de840f6a70c1919736cd6df as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/ee79d1767de840f6a70c1919736cd6df 2024-12-09T01:42:06,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/ee79d1767de840f6a70c1919736cd6df, entries=150, sequenceid=666, filesize=12.0 K 2024-12-09T01:42:06,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/0b5250342e9842d682d5808329202c7f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/0b5250342e9842d682d5808329202c7f 2024-12-09T01:42:06,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/0b5250342e9842d682d5808329202c7f, entries=150, sequenceid=666, filesize=12.0 K 2024-12-09T01:42:06,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~279.89 KB/286608, currentSize=87.22 KB/89310 for bb023e4ee9cc2de03187f3dff0a271ab in 902ms, sequenceid=666, compaction requested=true 2024-12-09T01:42:06,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:06,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:42:06,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:06,177 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:06,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:42:06,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:06,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:42:06,177 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:06,177 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:06,180 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41989 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:06,180 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:06,180 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/A is initiating minor compaction (all files) 2024-12-09T01:42:06,180 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/B is initiating minor compaction (all files) 2024-12-09T01:42:06,180 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/A in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:06,180 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/B in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:06,180 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/babd66918b8e42ce97f417d472216c20, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/b1982169ca5e417b9334456d15e41b3b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/9fb5e03323964c7c847627ac55e099c1] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=41.0 K 2024-12-09T01:42:06,180 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/fbfd255687f846b582208f65bd75cd43, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/046557ea4e9e46e5a84856c009d151ea, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/ee79d1767de840f6a70c1919736cd6df] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=36.2 K 2024-12-09T01:42:06,181 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting babd66918b8e42ce97f417d472216c20, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733708524576 2024-12-09T01:42:06,181 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting fbfd255687f846b582208f65bd75cd43, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733708524576 2024-12-09T01:42:06,182 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1982169ca5e417b9334456d15e41b3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1733708524617 2024-12-09T01:42:06,182 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 046557ea4e9e46e5a84856c009d151ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1733708524617 2024-12-09T01:42:06,183 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting ee79d1767de840f6a70c1919736cd6df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1733708525258 2024-12-09T01:42:06,183 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fb5e03323964c7c847627ac55e099c1, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1733708525258 2024-12-09T01:42:06,196 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#B#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:06,197 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/0aeefa7f4f314efe90733923f600e5de is 50, key is test_row_0/B:col10/1733708525258/Put/seqid=0 2024-12-09T01:42:06,201 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#A#compaction#241 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:06,201 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/bb38bdc20d4c4eb78a09175a6ba33f27 is 50, key is test_row_0/A:col10/1733708525258/Put/seqid=0 2024-12-09T01:42:06,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-12-09T01:42:06,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:06,210 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-12-09T01:42:06,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742108_1284 (size=12609) 2024-12-09T01:42:06,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:06,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:06,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:06,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:06,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:06,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:06,217 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/0aeefa7f4f314efe90733923f600e5de as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/0aeefa7f4f314efe90733923f600e5de 2024-12-09T01:42:06,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/a78553a5c9cd4f6688fd4ac37a6064e0 is 50, key is test_row_0/A:col10/1733708525303/Put/seqid=0 2024-12-09T01:42:06,224 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/B of bb023e4ee9cc2de03187f3dff0a271ab into 0aeefa7f4f314efe90733923f600e5de(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:06,224 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:06,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742109_1285 (size=12609) 2024-12-09T01:42:06,225 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/B, priority=13, startTime=1733708526177; duration=0sec 2024-12-09T01:42:06,225 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:06,225 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:B 2024-12-09T01:42:06,225 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:06,227 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37109 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:06,227 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/C is initiating minor compaction (all files) 2024-12-09T01:42:06,227 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/C in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:06,227 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/67afcbf187cc46f483590567fb671899, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/c09c0e086ff14789a320ec66acd4be27, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/0b5250342e9842d682d5808329202c7f] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=36.2 K 2024-12-09T01:42:06,227 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 67afcbf187cc46f483590567fb671899, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1733708524576 2024-12-09T01:42:06,228 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting c09c0e086ff14789a320ec66acd4be27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1733708524617 2024-12-09T01:42:06,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742110_1286 (size=12301) 2024-12-09T01:42:06,229 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b5250342e9842d682d5808329202c7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1733708525258 2024-12-09T01:42:06,232 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/bb38bdc20d4c4eb78a09175a6ba33f27 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/bb38bdc20d4c4eb78a09175a6ba33f27 2024-12-09T01:42:06,239 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#C#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:06,240 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/5e6fd3c6a8dc4b4abaf5f6982e9f6db2 is 50, key is test_row_0/C:col10/1733708525258/Put/seqid=0 2024-12-09T01:42:06,245 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/A of bb023e4ee9cc2de03187f3dff0a271ab into bb38bdc20d4c4eb78a09175a6ba33f27(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:06,245 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:06,245 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/A, priority=13, startTime=1733708526177; duration=0sec 2024-12-09T01:42:06,245 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:06,245 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:A 2024-12-09T01:42:06,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742111_1287 (size=12609) 2024-12-09T01:42:06,265 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/5e6fd3c6a8dc4b4abaf5f6982e9f6db2 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/5e6fd3c6a8dc4b4abaf5f6982e9f6db2 2024-12-09T01:42:06,272 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/C of bb023e4ee9cc2de03187f3dff0a271ab into 5e6fd3c6a8dc4b4abaf5f6982e9f6db2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:06,272 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:06,272 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/C, priority=13, startTime=1733708526177; duration=0sec 2024-12-09T01:42:06,272 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:06,272 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:C 2024-12-09T01:42:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:06,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:06,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708586460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:06,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:06,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:06,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708586461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:06,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:06,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708586464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:06,468 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:06,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708586465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:06,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:06,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708586465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:06,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708586573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708586574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708586574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:06,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:06,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708586574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:06,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:06,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:06,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708586574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:06,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:06,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,578 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:06,629 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/a78553a5c9cd4f6688fd4ac37a6064e0 2024-12-09T01:42:06,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/ae1ecf9497e3403fac6d45ad43b9ef22 is 50, key is test_row_0/B:col10/1733708525303/Put/seqid=0 2024-12-09T01:42:06,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742112_1288 (size=12301) 2024-12-09T01:42:06,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708586783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,785 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:06,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:06,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708586784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:06,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:06,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708586784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:06,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,788 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:06,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:06,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708586784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:06,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:06,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:06,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708586784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:06,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:06,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:06,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:07,044 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/ae1ecf9497e3403fac6d45ad43b9ef22 2024-12-09T01:42:07,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/6a8425f57c3c4d43a8091e0ab7bf5f31 is 50, key is test_row_0/C:col10/1733708525303/Put/seqid=0 2024-12-09T01:42:07,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742113_1289 (size=12301) 2024-12-09T01:42:07,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708587094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:07,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:07,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708587094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:07,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:07,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:07,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708587094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:07,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:07,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708587094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:07,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:07,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708587094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:07,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:07,462 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/6a8425f57c3c4d43a8091e0ab7bf5f31 2024-12-09T01:42:07,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/a78553a5c9cd4f6688fd4ac37a6064e0 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/a78553a5c9cd4f6688fd4ac37a6064e0 2024-12-09T01:42:07,478 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/a78553a5c9cd4f6688fd4ac37a6064e0, entries=150, sequenceid=682, filesize=12.0 K 2024-12-09T01:42:07,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/ae1ecf9497e3403fac6d45ad43b9ef22 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/ae1ecf9497e3403fac6d45ad43b9ef22 2024-12-09T01:42:07,484 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/ae1ecf9497e3403fac6d45ad43b9ef22, entries=150, sequenceid=682, filesize=12.0 K 2024-12-09T01:42:07,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/6a8425f57c3c4d43a8091e0ab7bf5f31 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/6a8425f57c3c4d43a8091e0ab7bf5f31 2024-12-09T01:42:07,489 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/6a8425f57c3c4d43a8091e0ab7bf5f31, entries=150, sequenceid=682, filesize=12.0 K 2024-12-09T01:42:07,490 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for bb023e4ee9cc2de03187f3dff0a271ab in 1280ms, sequenceid=682, compaction requested=false 2024-12-09T01:42:07,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:07,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:07,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-12-09T01:42:07,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-12-09T01:42:07,493 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-12-09T01:42:07,493 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7540 sec 2024-12-09T01:42:07,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 3.7600 sec 2024-12-09T01:42:07,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:07,607 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T01:42:07,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:07,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:07,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:07,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:07,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:07,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:07,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/a6153bfc99ec4eaf8c7b51d500b21ec7 is 50, key is test_row_0/A:col10/1733708527606/Put/seqid=0 2024-12-09T01:42:07,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:07,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708587619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:07,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:07,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708587620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:07,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,625 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:07,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:07,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708587621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:07,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:07,627 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:07,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708587622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:07,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:07,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:07,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708587622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:07,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:07,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742114_1290 (size=14741) 2024-12-09T01:42:07,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:07,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708587734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:07,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:07,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,737 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:07,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708587734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:07,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:07,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:07,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708587734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:07,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:07,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:07,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708587734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:07,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:07,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708587734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:07,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:07,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:07,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-09T01:42:07,904 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:07,904 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:42:07,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-12-09T01:42:07,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T01:42:07,907 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:42:07,908 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:42:07,908 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:42:07,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:07,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708587943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:07,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:07,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708587943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:07,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:07,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:07,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708587943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:07,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,947 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:07,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708587944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:07,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708587944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:07,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:08,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T01:42:08,033 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/a6153bfc99ec4eaf8c7b51d500b21ec7 2024-12-09T01:42:08,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/744493e095dd47cfb6b554a36c7ac0e4 is 50, key is test_row_0/B:col10/1733708527606/Put/seqid=0 2024-12-09T01:42:08,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742115_1291 (size=12301) 2024-12-09T01:42:08,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/744493e095dd47cfb6b554a36c7ac0e4 2024-12-09T01:42:08,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T01:42:08,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:08,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:08,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:08,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:08,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:08,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:08,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/165509d453394ef5bac7e1e18e275cc4 is 50, key is test_row_0/C:col10/1733708527606/Put/seqid=0 2024-12-09T01:42:08,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742116_1292 (size=12301) 2024-12-09T01:42:08,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T01:42:08,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:08,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:08,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:08,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:08,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:08,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:08,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T01:42:08,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:08,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708588253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:08,257 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:08,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708588253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:08,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:08,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708588253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:08,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:08,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708588253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:08,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:08,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708588254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,259 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:08,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T01:42:08,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:08,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:08,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:08,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:08,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:08,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:08,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=705 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/165509d453394ef5bac7e1e18e275cc4 2024-12-09T01:42:08,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/a6153bfc99ec4eaf8c7b51d500b21ec7 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/a6153bfc99ec4eaf8c7b51d500b21ec7 2024-12-09T01:42:08,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/a6153bfc99ec4eaf8c7b51d500b21ec7, entries=200, sequenceid=705, filesize=14.4 K 2024-12-09T01:42:08,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/744493e095dd47cfb6b554a36c7ac0e4 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/744493e095dd47cfb6b554a36c7ac0e4 2024-12-09T01:42:08,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/744493e095dd47cfb6b554a36c7ac0e4, entries=150, sequenceid=705, filesize=12.0 K 2024-12-09T01:42:08,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/165509d453394ef5bac7e1e18e275cc4 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/165509d453394ef5bac7e1e18e275cc4 2024-12-09T01:42:08,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/165509d453394ef5bac7e1e18e275cc4, entries=150, sequenceid=705, filesize=12.0 K 2024-12-09T01:42:08,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for bb023e4ee9cc2de03187f3dff0a271ab in 880ms, sequenceid=705, compaction requested=true 2024-12-09T01:42:08,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:08,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:42:08,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:08,487 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:08,487 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:08,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:42:08,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:08,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:42:08,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:08,488 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:08,488 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/B is initiating minor compaction (all files) 2024-12-09T01:42:08,489 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/B in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:08,489 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/0aeefa7f4f314efe90733923f600e5de, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/ae1ecf9497e3403fac6d45ad43b9ef22, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/744493e095dd47cfb6b554a36c7ac0e4] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=36.3 K 2024-12-09T01:42:08,489 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39651 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:08,489 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/A is initiating minor compaction (all files) 2024-12-09T01:42:08,489 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/A in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:08,489 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/bb38bdc20d4c4eb78a09175a6ba33f27, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/a78553a5c9cd4f6688fd4ac37a6064e0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/a6153bfc99ec4eaf8c7b51d500b21ec7] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=38.7 K 2024-12-09T01:42:08,489 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb38bdc20d4c4eb78a09175a6ba33f27, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1733708525258 2024-12-09T01:42:08,489 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aeefa7f4f314efe90733923f600e5de, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1733708525258 2024-12-09T01:42:08,490 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting ae1ecf9497e3403fac6d45ad43b9ef22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=682, earliestPutTs=1733708525288 2024-12-09T01:42:08,490 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting a78553a5c9cd4f6688fd4ac37a6064e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=682, earliestPutTs=1733708525288 2024-12-09T01:42:08,490 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 744493e095dd47cfb6b554a36c7ac0e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=705, earliestPutTs=1733708526463 2024-12-09T01:42:08,490 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6153bfc99ec4eaf8c7b51d500b21ec7, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=705, earliestPutTs=1733708526463 2024-12-09T01:42:08,498 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#B#compaction#250 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:08,498 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#A#compaction#249 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:08,498 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/f93748e8455b4b7f83d0924faa89a499 is 50, key is test_row_0/B:col10/1733708527606/Put/seqid=0 2024-12-09T01:42:08,498 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/18c7bce278e6401286a0725c251d9b72 is 50, key is test_row_0/A:col10/1733708527606/Put/seqid=0 2024-12-09T01:42:08,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-12-09T01:42:08,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:08,521 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T01:42:08,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:08,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:08,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:08,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:08,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:08,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:08,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/10464651de07473680c03e7c5a38a908 is 50, key is test_row_0/A:col10/1733708527620/Put/seqid=0 2024-12-09T01:42:08,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T01:42:08,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742118_1294 (size=12711) 2024-12-09T01:42:08,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742117_1293 (size=12711) 2024-12-09T01:42:08,551 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/18c7bce278e6401286a0725c251d9b72 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/18c7bce278e6401286a0725c251d9b72 2024-12-09T01:42:08,554 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/f93748e8455b4b7f83d0924faa89a499 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/f93748e8455b4b7f83d0924faa89a499 2024-12-09T01:42:08,559 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/A of bb023e4ee9cc2de03187f3dff0a271ab into 18c7bce278e6401286a0725c251d9b72(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:08,559 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:08,559 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/A, priority=13, startTime=1733708528487; duration=0sec 2024-12-09T01:42:08,559 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:08,559 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:A 2024-12-09T01:42:08,559 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:08,561 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37211 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:08,561 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/C is initiating minor compaction (all files) 2024-12-09T01:42:08,561 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/C in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:08,561 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/5e6fd3c6a8dc4b4abaf5f6982e9f6db2, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/6a8425f57c3c4d43a8091e0ab7bf5f31, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/165509d453394ef5bac7e1e18e275cc4] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=36.3 K 2024-12-09T01:42:08,561 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e6fd3c6a8dc4b4abaf5f6982e9f6db2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=666, earliestPutTs=1733708525258 2024-12-09T01:42:08,562 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a8425f57c3c4d43a8091e0ab7bf5f31, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=682, earliestPutTs=1733708525288 2024-12-09T01:42:08,562 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 165509d453394ef5bac7e1e18e275cc4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=705, earliestPutTs=1733708526463 2024-12-09T01:42:08,562 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/B of bb023e4ee9cc2de03187f3dff0a271ab into f93748e8455b4b7f83d0924faa89a499(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:08,562 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:08,562 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/B, priority=13, startTime=1733708528487; duration=0sec 2024-12-09T01:42:08,562 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:08,562 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:B 2024-12-09T01:42:08,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742119_1295 (size=12301) 2024-12-09T01:42:08,569 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#C#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:08,570 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/c4008d26de184e05a1b9959de7b25981 is 50, key is test_row_0/C:col10/1733708527606/Put/seqid=0 2024-12-09T01:42:08,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742120_1296 (size=12711) 2024-12-09T01:42:08,578 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/c4008d26de184e05a1b9959de7b25981 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/c4008d26de184e05a1b9959de7b25981 2024-12-09T01:42:08,583 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/C of bb023e4ee9cc2de03187f3dff0a271ab into c4008d26de184e05a1b9959de7b25981(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:08,583 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:08,583 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/C, priority=13, startTime=1733708528487; duration=0sec 2024-12-09T01:42:08,583 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:08,583 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:C 2024-12-09T01:42:08,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:08,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:08,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:08,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:08,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708588791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:08,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708588791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:08,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:08,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,801 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:08,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:08,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708588793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:08,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:08,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708588800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:08,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:08,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708588801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:08,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:08,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708588903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:08,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708588904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:08,916 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708588913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:08,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:08,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708588913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:08,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:08,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,917 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708588913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:08,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:08,918 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:08,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:08,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:08,966 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=718 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/10464651de07473680c03e7c5a38a908 2024-12-09T01:42:08,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/d14918fad71f490c9e65d1f778953cac is 50, key is test_row_0/B:col10/1733708527620/Put/seqid=0 2024-12-09T01:42:09,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742121_1297 (size=12301) 2024-12-09T01:42:09,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T01:42:09,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:09,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708589113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:09,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:09,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:09,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708589113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:09,116 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:09,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:09,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:09,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:09,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708589123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:09,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:09,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:09,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:09,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708589124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:09,126 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:09,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:09,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:09,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708589124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:09,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:09,419 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=718 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/d14918fad71f490c9e65d1f778953cac 2024-12-09T01:42:09,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708589424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:09,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:09,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:09,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:09,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708589424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:09,428 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:09,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/00a48995faca4aa3b360003e536e2a14 is 50, key is test_row_0/C:col10/1733708527620/Put/seqid=0 2024-12-09T01:42:09,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742122_1298 (size=12301) 2024-12-09T01:42:09,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708589433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:09,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:09,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:09,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:09,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708589434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:09,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708589434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:09,439 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:09,837 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=718 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/00a48995faca4aa3b360003e536e2a14 2024-12-09T01:42:09,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/10464651de07473680c03e7c5a38a908 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/10464651de07473680c03e7c5a38a908 2024-12-09T01:42:09,848 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/10464651de07473680c03e7c5a38a908, entries=150, sequenceid=718, filesize=12.0 K 2024-12-09T01:42:09,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/d14918fad71f490c9e65d1f778953cac as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/d14918fad71f490c9e65d1f778953cac 2024-12-09T01:42:09,857 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/d14918fad71f490c9e65d1f778953cac, entries=150, sequenceid=718, filesize=12.0 K 2024-12-09T01:42:09,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/00a48995faca4aa3b360003e536e2a14 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/00a48995faca4aa3b360003e536e2a14 2024-12-09T01:42:09,867 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/00a48995faca4aa3b360003e536e2a14, entries=150, sequenceid=718, filesize=12.0 K 2024-12-09T01:42:09,868 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for bb023e4ee9cc2de03187f3dff0a271ab in 1347ms, sequenceid=718, compaction requested=false 2024-12-09T01:42:09,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:09,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:09,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-12-09T01:42:09,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-12-09T01:42:09,873 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-12-09T01:42:09,873 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9620 sec 2024-12-09T01:42:09,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.9690 sec 2024-12-09T01:42:09,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708528786, max timestamp=1733708529937Num uniques 859; 2024-12-09T01:42:09,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:09,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:42:09,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708528786, max timestamp=1733708529937Num uniques 859; 2024-12-09T01:42:09,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:42:09,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-12-09T01:42:09,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708528786, max timestamp=1733708529937Num uniques 859; 2024-12-09T01:42:09,940 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:42:09,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:09,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:09,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:09,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:09,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:09,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:09,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/6dd7527c155b45d7bd4d2a81e75111a0 is 50, key is test_row_0/A:col10/1733708528786/Put/seqid=0 2024-12-09T01:42:09,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742123_1299 (size=14741) 2024-12-09T01:42:09,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=743 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/6dd7527c155b45d7bd4d2a81e75111a0 2024-12-09T01:42:09,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/a8dd8d20fd7b42fb9e0cad2dda1318ad is 50, key is test_row_0/B:col10/1733708528786/Put/seqid=0 2024-12-09T01:42:09,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708529963, max timestamp=1733708529975Num uniques 859; 2024-12-09T01:42:09,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:42:09,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708529963, max timestamp=1733708529975Num uniques 859; 2024-12-09T01:42:09,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708529963, max timestamp=1733708529975Num uniques 859; 2024-12-09T01:42:09,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:42:09,982 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:42:10,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742124_1300 (size=12301) 2024-12-09T01:42:10,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:10,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708590007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:10,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:10,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708590008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:10,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733708530003, max timestamp=1733708530015Num uniques 1068; 2024-12-09T01:42:10,020 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-09T01:42:10,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733708530003, max timestamp=1733708530015Num uniques 1068; 2024-12-09T01:42:10,021 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-09T01:42:10,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733708530003, max timestamp=1733708530015Num uniques 1068; 2024-12-09T01:42:10,022 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-12-09T01:42:10,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-12-09T01:42:10,054 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:10,055 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:42:10,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-12-09T01:42:10,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T01:42:10,057 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:42:10,058 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:42:10,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:42:10,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733708530056, max timestamp=1733708530068Num uniques 1077; 2024-12-09T01:42:10,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-09T01:42:10,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733708530056, max timestamp=1733708530068Num uniques 1077; 2024-12-09T01:42:10,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1733708530056, max timestamp=1733708530068Num uniques 1077; 2024-12-09T01:42:10,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-09T01:42:10,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=12682, suffix heap size=139976,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=277 2024-12-09T01:42:10,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:10,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708590119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:10,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733708530093, max timestamp=1733708530124Num uniques 1136; 2024-12-09T01:42:10,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-09T01:42:10,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733708530093, max timestamp=1733708530124Num uniques 1136; 2024-12-09T01:42:10,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-09T01:42:10,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,129 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:10,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1733708530093, max timestamp=1733708530124Num uniques 1136; 2024-12-09T01:42:10,131 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52026, new segment data size=8516, suffix heap size=146112,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1136, new segment cells count=186 2024-12-09T01:42:10,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T01:42:10,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733708530135, max timestamp=1733708530183Num uniques 1045; 2024-12-09T01:42:10,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-09T01:42:10,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733708530135, max timestamp=1733708530183Num uniques 1045; 2024-12-09T01:42:10,188 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-09T01:42:10,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1733708530135, max timestamp=1733708530183Num uniques 1045; 2024-12-09T01:42:10,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-12-09T01:42:10,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T01:42:10,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:10,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:10,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:10,212 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:10,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:10,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:10,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=254, cellSize=11630, totalHeapSize=26656, min timestamp=1733708530234, max timestamp=1733708530247Num uniques 1054; 2024-12-09T01:42:10,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=11630, suffix heap size=137584,new segment heap size=26656 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=254 2024-12-09T01:42:10,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733708530234, max timestamp=1733708530247Num uniques 1054; 2024-12-09T01:42:10,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-09T01:42:10,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1733708530234, max timestamp=1733708530247Num uniques 1054; 2024-12-09T01:42:10,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-12-09T01:42:10,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=174, cellSize=7964, totalHeapSize=18336, min timestamp=1733708530263, max timestamp=1733708530287Num uniques 1013; 2024-12-09T01:42:10,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7964, suffix heap size=133320,new segment heap size=18336 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=174 2024-12-09T01:42:10,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733708530263, max timestamp=1733708530287Num uniques 1013; 2024-12-09T01:42:10,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-12-09T01:42:10,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1733708530263, max timestamp=1733708530287Num uniques 1113; 2024-12-09T01:42:10,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50968, new segment data size=9748, suffix heap size=143720,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1113, new segment cells count=213 2024-12-09T01:42:10,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T01:42:10,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:10,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:10,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:10,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:10,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:10,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:10,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T01:42:10,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733708530360, max timestamp=1733708530373Num uniques 1072; 2024-12-09T01:42:10,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-12-09T01:42:10,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733708530360, max timestamp=1733708530373Num uniques 1072; 2024-12-09T01:42:10,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1733708530360, max timestamp=1733708530373Num uniques 1033; 2024-12-09T01:42:10,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-12-09T01:42:10,381 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47308, new segment data size=7872, suffix heap size=135400,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1033, new segment cells count=172 2024-12-09T01:42:10,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=743 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/a8dd8d20fd7b42fb9e0cad2dda1318ad 2024-12-09T01:42:10,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/e36807d918144bd58732cdb83d44e3aa is 50, key is test_row_0/C:col10/1733708528786/Put/seqid=0 2024-12-09T01:42:10,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742125_1301 (size=12301) 2024-12-09T01:42:10,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=743 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/e36807d918144bd58732cdb83d44e3aa 2024-12-09T01:42:10,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/6dd7527c155b45d7bd4d2a81e75111a0 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/6dd7527c155b45d7bd4d2a81e75111a0 2024-12-09T01:42:10,427 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/6dd7527c155b45d7bd4d2a81e75111a0, entries=200, sequenceid=743, filesize=14.4 K 2024-12-09T01:42:10,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/a8dd8d20fd7b42fb9e0cad2dda1318ad as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/a8dd8d20fd7b42fb9e0cad2dda1318ad 2024-12-09T01:42:10,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/a8dd8d20fd7b42fb9e0cad2dda1318ad, entries=150, sequenceid=743, filesize=12.0 K 2024-12-09T01:42:10,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/e36807d918144bd58732cdb83d44e3aa as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/e36807d918144bd58732cdb83d44e3aa 2024-12-09T01:42:10,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/e36807d918144bd58732cdb83d44e3aa, entries=150, sequenceid=743, filesize=12.0 K 2024-12-09T01:42:10,453 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=107.34 KB/109920 for bb023e4ee9cc2de03187f3dff0a271ab in 512ms, sequenceid=743, compaction requested=true 2024-12-09T01:42:10,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:10,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:42:10,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:10,453 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:10,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:42:10,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:10,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:42:10,453 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:10,453 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:10,454 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39753 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:10,454 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/A is initiating minor compaction (all files) 2024-12-09T01:42:10,454 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/A in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:10,454 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/18c7bce278e6401286a0725c251d9b72, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/10464651de07473680c03e7c5a38a908, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/6dd7527c155b45d7bd4d2a81e75111a0] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=38.8 K 2024-12-09T01:42:10,454 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:10,454 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/B is initiating minor compaction (all files) 2024-12-09T01:42:10,454 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/B in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:10,455 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/f93748e8455b4b7f83d0924faa89a499, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/d14918fad71f490c9e65d1f778953cac, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/a8dd8d20fd7b42fb9e0cad2dda1318ad] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=36.4 K 2024-12-09T01:42:10,455 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18c7bce278e6401286a0725c251d9b72, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=705, earliestPutTs=1733708526463 2024-12-09T01:42:10,455 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting f93748e8455b4b7f83d0924faa89a499, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=705, earliestPutTs=1733708526463 2024-12-09T01:42:10,455 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10464651de07473680c03e7c5a38a908, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=718, earliestPutTs=1733708527619 2024-12-09T01:42:10,455 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting d14918fad71f490c9e65d1f778953cac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=718, earliestPutTs=1733708527619 2024-12-09T01:42:10,455 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dd7527c155b45d7bd4d2a81e75111a0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=743, earliestPutTs=1733708528786 2024-12-09T01:42:10,456 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting a8dd8d20fd7b42fb9e0cad2dda1318ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=743, earliestPutTs=1733708528786 2024-12-09T01:42:10,471 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:10,471 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/cd4f17e9b834494ca52ead3bc5b68a94 is 50, key is test_row_0/A:col10/1733708528786/Put/seqid=0 2024-12-09T01:42:10,482 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#B#compaction#259 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:10,482 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/0797cf4f1d504a93b7b2f4cc05bfcec9 is 50, key is test_row_0/B:col10/1733708528786/Put/seqid=0 2024-12-09T01:42:10,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742126_1302 (size=12813) 2024-12-09T01:42:10,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742127_1303 (size=12813) 2024-12-09T01:42:10,497 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/cd4f17e9b834494ca52ead3bc5b68a94 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/cd4f17e9b834494ca52ead3bc5b68a94 2024-12-09T01:42:10,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:10,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=114.05 KB heapSize=324 KB 2024-12-09T01:42:10,509 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:10,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:10,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:10,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:10,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:10,510 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:10,517 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/A of bb023e4ee9cc2de03187f3dff0a271ab into cd4f17e9b834494ca52ead3bc5b68a94(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:10,517 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:10,517 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/A, priority=13, startTime=1733708530453; duration=0sec 2024-12-09T01:42:10,517 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:10,517 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:A 2024-12-09T01:42:10,517 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:10,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T01:42:10,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:10,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:10,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:10,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:10,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:10,523 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:10,523 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/C is initiating minor compaction (all files) 2024-12-09T01:42:10,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:10,523 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/C in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:10,523 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/c4008d26de184e05a1b9959de7b25981, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/00a48995faca4aa3b360003e536e2a14, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/e36807d918144bd58732cdb83d44e3aa] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=36.4 K 2024-12-09T01:42:10,524 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4008d26de184e05a1b9959de7b25981, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=705, earliestPutTs=1733708526463 2024-12-09T01:42:10,524 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00a48995faca4aa3b360003e536e2a14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=718, earliestPutTs=1733708527619 2024-12-09T01:42:10,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/06512df454924665943345567286c080 is 50, key is test_row_0/A:col10/1733708530452/Put/seqid=0 2024-12-09T01:42:10,530 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting e36807d918144bd58732cdb83d44e3aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=743, earliestPutTs=1733708528786 2024-12-09T01:42:10,561 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#C#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:10,561 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/78910a3e966347daafe8e2d2856e2fa0 is 50, key is test_row_0/C:col10/1733708528786/Put/seqid=0 2024-12-09T01:42:10,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:10,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708590549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:10,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:10,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708590551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:10,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:10,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:10,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:10,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708590543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:10,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,568 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:10,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:10,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708590560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:10,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:10,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708590560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,577 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:10,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742128_1304 (size=12301) 2024-12-09T01:42:10,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=899 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/06512df454924665943345567286c080 2024-12-09T01:42:10,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742129_1305 (size=12813) 2024-12-09T01:42:10,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/3b73c4c2befb4b2b893c54bd27e6e3f2 is 50, key is test_row_0/B:col10/1733708530452/Put/seqid=0 2024-12-09T01:42:10,635 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/78910a3e966347daafe8e2d2856e2fa0 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/78910a3e966347daafe8e2d2856e2fa0 2024-12-09T01:42:10,640 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/C of bb023e4ee9cc2de03187f3dff0a271ab into 78910a3e966347daafe8e2d2856e2fa0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:10,640 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:10,640 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/C, priority=13, startTime=1733708530453; duration=0sec 2024-12-09T01:42:10,641 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:10,641 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:C 2024-12-09T01:42:10,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742130_1306 (size=12301) 2024-12-09T01:42:10,674 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=899 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/3b73c4c2befb4b2b893c54bd27e6e3f2 2024-12-09T01:42:10,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T01:42:10,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:10,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:10,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:10,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:10,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:10,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:10,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:10,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708590673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:10,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:10,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708590673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:10,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:10,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708590673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:10,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,682 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:10,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:10,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:10,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T01:42:10,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:10,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708590684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:10,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:10,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:10,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708590684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:10,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,695 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:10,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/47a846406c4f4d07b7e4ce7e97a6333f is 50, key is test_row_0/C:col10/1733708530452/Put/seqid=0 2024-12-09T01:42:10,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742131_1307 (size=12301) 2024-12-09T01:42:10,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=899 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/47a846406c4f4d07b7e4ce7e97a6333f 2024-12-09T01:42:10,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/06512df454924665943345567286c080 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/06512df454924665943345567286c080 2024-12-09T01:42:10,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/06512df454924665943345567286c080, entries=150, sequenceid=899, filesize=12.0 K 2024-12-09T01:42:10,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/3b73c4c2befb4b2b893c54bd27e6e3f2 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/3b73c4c2befb4b2b893c54bd27e6e3f2 2024-12-09T01:42:10,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/3b73c4c2befb4b2b893c54bd27e6e3f2, entries=150, sequenceid=899, filesize=12.0 K 2024-12-09T01:42:10,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/47a846406c4f4d07b7e4ce7e97a6333f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/47a846406c4f4d07b7e4ce7e97a6333f 2024-12-09T01:42:10,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/47a846406c4f4d07b7e4ce7e97a6333f, entries=150, sequenceid=899, filesize=12.0 K 2024-12-09T01:42:10,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~363.80 KB/372528, currentSize=60.38 KB/61830 for bb023e4ee9cc2de03187f3dff0a271ab in 242ms, sequenceid=899, compaction requested=false 2024-12-09T01:42:10,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:10,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-12-09T01:42:10,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:10,831 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T01:42:10,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:10,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:10,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:10,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:10,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:10,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:10,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/253f010e243046cfb9616b7a71ea186c is 50, key is test_row_0/A:col10/1733708530543/Put/seqid=0 2024-12-09T01:42:10,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742132_1308 (size=12301) 2024-12-09T01:42:10,879 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=913 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/253f010e243046cfb9616b7a71ea186c 2024-12-09T01:42:10,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/9843148eda3847b8acfdb48443e68502 is 50, key is test_row_0/B:col10/1733708530543/Put/seqid=0 2024-12-09T01:42:10,898 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/0797cf4f1d504a93b7b2f4cc05bfcec9 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/0797cf4f1d504a93b7b2f4cc05bfcec9 2024-12-09T01:42:10,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:10,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:10,903 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/B of bb023e4ee9cc2de03187f3dff0a271ab into 0797cf4f1d504a93b7b2f4cc05bfcec9(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:10,903 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:10,903 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/B, priority=13, startTime=1733708530453; duration=0sec 2024-12-09T01:42:10,904 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:10,904 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:B 2024-12-09T01:42:10,943 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11854, totalHeapSize=27176, min timestamp=1733708530927, max timestamp=1733708530940Num uniques 859; 2024-12-09T01:42:10,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=11854, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:42:10,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708530935, max timestamp=1733708530940Num uniques 859; 2024-12-09T01:42:10,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:42:10,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708530935, max timestamp=1733708530940Num uniques 859; 2024-12-09T01:42:10,947 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:42:10,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742133_1309 (size=12301) 2024-12-09T01:42:10,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:10,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708590971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:10,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:10,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:10,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708590973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:10,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:10,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708590979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:10,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:10,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708590982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:10,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:10,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:10,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708590980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:10,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:10,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:10,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708591084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708591093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708591093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708591093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708591094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,099 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T01:42:11,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708591303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,308 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708591304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708591304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708591304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708591304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,310 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,351 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=913 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/9843148eda3847b8acfdb48443e68502 2024-12-09T01:42:11,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/9804d761c58e48ffa74f652ed5abfa85 is 50, key is test_row_0/C:col10/1733708530543/Put/seqid=0 2024-12-09T01:42:11,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742134_1310 (size=12301) 2024-12-09T01:42:11,426 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=913 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/9804d761c58e48ffa74f652ed5abfa85 2024-12-09T01:42:11,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/253f010e243046cfb9616b7a71ea186c as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/253f010e243046cfb9616b7a71ea186c 2024-12-09T01:42:11,437 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/253f010e243046cfb9616b7a71ea186c, entries=150, sequenceid=913, filesize=12.0 K 2024-12-09T01:42:11,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/9843148eda3847b8acfdb48443e68502 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/9843148eda3847b8acfdb48443e68502 2024-12-09T01:42:11,443 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/9843148eda3847b8acfdb48443e68502, entries=150, sequenceid=913, filesize=12.0 K 2024-12-09T01:42:11,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/9804d761c58e48ffa74f652ed5abfa85 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/9804d761c58e48ffa74f652ed5abfa85 2024-12-09T01:42:11,449 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/9804d761c58e48ffa74f652ed5abfa85, entries=150, sequenceid=913, filesize=12.0 K 2024-12-09T01:42:11,452 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=125.23 KB/128240 for bb023e4ee9cc2de03187f3dff0a271ab in 621ms, sequenceid=913, compaction requested=true 2024-12-09T01:42:11,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:11,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:11,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-12-09T01:42:11,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-12-09T01:42:11,456 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-12-09T01:42:11,456 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3960 sec 2024-12-09T01:42:11,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.4020 sec 2024-12-09T01:42:11,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:11,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=138.65 KB heapSize=395.14 KB 2024-12-09T01:42:11,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:11,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T01:42:11,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:11,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T01:42:11,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:11,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T01:42:11,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/fef93d4987324883bbf9dd569b9bd9ec is 50, key is test_row_0/A:col10/1733708530968/Put/seqid=0 2024-12-09T01:42:11,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708591638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708591639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708591646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708591648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708591648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,659 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742135_1311 (size=12301) 2024-12-09T01:42:11,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708591768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708591768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708591767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708591768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,775 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708591768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708591984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,990 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708591984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,990 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,991 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708591984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708591984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,992 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:11,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:11,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708591984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:11,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:11,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:12,099 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=952 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/fef93d4987324883bbf9dd569b9bd9ec 2024-12-09T01:42:12,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/55e23cafd2684477b833475aceefd25a is 50, key is test_row_0/B:col10/1733708530968/Put/seqid=0 2024-12-09T01:42:12,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742136_1312 (size=12301) 2024-12-09T01:42:12,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-12-09T01:42:12,204 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:12,204 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:42:12,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-12-09T01:42:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T01:42:12,207 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:42:12,208 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:42:12,208 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:42:12,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708592293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:12,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:12,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708592293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:12,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:12,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:12,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:12,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:12,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708592294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:12,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:12,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:12,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:12,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:12,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:12,299 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:12,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708592304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:12,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:12,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:12,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:12,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:12,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708592304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:12,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:12,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T01:42:12,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T01:42:12,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:12,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:12,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:12,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:12,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:12,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:12,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T01:42:12,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:12,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:12,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:12,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:12,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T01:42:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:12,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=952 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/55e23cafd2684477b833475aceefd25a 2024-12-09T01:42:12,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/9234e3d7e09a4001a8a01794efe1798c is 50, key is test_row_0/C:col10/1733708530968/Put/seqid=0 2024-12-09T01:42:12,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742137_1313 (size=12301) 2024-12-09T01:42:12,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=952 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/9234e3d7e09a4001a8a01794efe1798c 2024-12-09T01:42:12,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/fef93d4987324883bbf9dd569b9bd9ec as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/fef93d4987324883bbf9dd569b9bd9ec 2024-12-09T01:42:12,622 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/fef93d4987324883bbf9dd569b9bd9ec, entries=150, sequenceid=952, filesize=12.0 K 2024-12-09T01:42:12,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/55e23cafd2684477b833475aceefd25a as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/55e23cafd2684477b833475aceefd25a 2024-12-09T01:42:12,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/55e23cafd2684477b833475aceefd25a, entries=150, sequenceid=952, filesize=12.0 K 2024-12-09T01:42:12,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/9234e3d7e09a4001a8a01794efe1798c as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/9234e3d7e09a4001a8a01794efe1798c 2024-12-09T01:42:12,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/9234e3d7e09a4001a8a01794efe1798c, entries=150, sequenceid=952, filesize=12.0 K 2024-12-09T01:42:12,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~145.36 KB/148850, heapSize ~415.72 KB/425696, currentSize=33.54 KB/34350 for bb023e4ee9cc2de03187f3dff0a271ab in 1005ms, sequenceid=952, compaction requested=true 2024-12-09T01:42:12,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:12,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:42:12,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:12,635 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:42:12,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:42:12,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:12,635 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:42:12,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:42:12,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:12,636 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:42:12,636 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/A is initiating minor compaction (all files) 2024-12-09T01:42:12,636 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/A in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:12,636 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/cd4f17e9b834494ca52ead3bc5b68a94, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/06512df454924665943345567286c080, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/253f010e243046cfb9616b7a71ea186c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/fef93d4987324883bbf9dd569b9bd9ec] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=48.6 K 2024-12-09T01:42:12,637 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:42:12,637 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/B is initiating minor compaction (all files) 2024-12-09T01:42:12,637 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/B in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:12,637 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/0797cf4f1d504a93b7b2f4cc05bfcec9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/3b73c4c2befb4b2b893c54bd27e6e3f2, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/9843148eda3847b8acfdb48443e68502, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/55e23cafd2684477b833475aceefd25a] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=48.6 K 2024-12-09T01:42:12,637 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd4f17e9b834494ca52ead3bc5b68a94, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=743, earliestPutTs=1733708528786 2024-12-09T01:42:12,637 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 0797cf4f1d504a93b7b2f4cc05bfcec9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=743, earliestPutTs=1733708528786 2024-12-09T01:42:12,638 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06512df454924665943345567286c080, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=899, earliestPutTs=1733708530449 2024-12-09T01:42:12,638 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b73c4c2befb4b2b893c54bd27e6e3f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=899, earliestPutTs=1733708530449 2024-12-09T01:42:12,638 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 253f010e243046cfb9616b7a71ea186c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=913, earliestPutTs=1733708530543 2024-12-09T01:42:12,638 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting fef93d4987324883bbf9dd569b9bd9ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=952, earliestPutTs=1733708530968 2024-12-09T01:42:12,638 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 9843148eda3847b8acfdb48443e68502, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=913, earliestPutTs=1733708530543 2024-12-09T01:42:12,639 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 55e23cafd2684477b833475aceefd25a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=952, earliestPutTs=1733708530968 2024-12-09T01:42:12,663 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#B#compaction#270 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:12,663 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/72d8980527004649b33758faefd2bbb4 is 50, key is test_row_0/B:col10/1733708530968/Put/seqid=0 2024-12-09T01:42:12,674 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#A#compaction#271 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:12,674 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/865c842be51249a189b7053d91a8dbd8 is 50, key is test_row_0/A:col10/1733708530968/Put/seqid=0 2024-12-09T01:42:12,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-12-09T01:42:12,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:12,677 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-09T01:42:12,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:12,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:12,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:12,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:12,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:12,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:12,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/50f3f59d7c554f1c86283edae7767759 is 50, key is test_row_0/A:col10/1733708531646/Put/seqid=0 2024-12-09T01:42:12,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742138_1314 (size=12949) 2024-12-09T01:42:12,741 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/72d8980527004649b33758faefd2bbb4 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/72d8980527004649b33758faefd2bbb4 2024-12-09T01:42:12,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742139_1315 (size=12949) 2024-12-09T01:42:12,751 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/B of bb023e4ee9cc2de03187f3dff0a271ab into 72d8980527004649b33758faefd2bbb4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:12,751 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:12,751 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/B, priority=12, startTime=1733708532635; duration=0sec 2024-12-09T01:42:12,752 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:12,752 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:B 2024-12-09T01:42:12,752 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/865c842be51249a189b7053d91a8dbd8 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/865c842be51249a189b7053d91a8dbd8 2024-12-09T01:42:12,752 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:42:12,756 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49716 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:42:12,761 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/C is initiating minor compaction (all files) 2024-12-09T01:42:12,761 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/C in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:12,761 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/78910a3e966347daafe8e2d2856e2fa0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/47a846406c4f4d07b7e4ce7e97a6333f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/9804d761c58e48ffa74f652ed5abfa85, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/9234e3d7e09a4001a8a01794efe1798c] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=48.6 K 2024-12-09T01:42:12,761 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/A of bb023e4ee9cc2de03187f3dff0a271ab into 865c842be51249a189b7053d91a8dbd8(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:12,761 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:12,761 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/A, priority=12, startTime=1733708532635; duration=0sec 2024-12-09T01:42:12,762 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 78910a3e966347daafe8e2d2856e2fa0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=743, earliestPutTs=1733708528786 2024-12-09T01:42:12,762 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:12,762 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:A 2024-12-09T01:42:12,763 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 47a846406c4f4d07b7e4ce7e97a6333f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=899, earliestPutTs=1733708530449 2024-12-09T01:42:12,763 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 9804d761c58e48ffa74f652ed5abfa85, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=913, earliestPutTs=1733708530543 2024-12-09T01:42:12,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742140_1316 (size=12301) 2024-12-09T01:42:12,764 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 9234e3d7e09a4001a8a01794efe1798c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=952, earliestPutTs=1733708530968 2024-12-09T01:42:12,765 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=960 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/50f3f59d7c554f1c86283edae7767759 2024-12-09T01:42:12,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/30cb3f8bfce7447ab604ec3639d1cb29 is 50, key is test_row_0/B:col10/1733708531646/Put/seqid=0 2024-12-09T01:42:12,791 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#C#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:12,791 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/4e7adf136c88469ba3c188379da1b2d2 is 50, key is test_row_0/C:col10/1733708530968/Put/seqid=0 2024-12-09T01:42:12,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742142_1318 (size=12949) 2024-12-09T01:42:12,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:12,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:12,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T01:42:12,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742141_1317 (size=12301) 2024-12-09T01:42:12,835 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=960 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/30cb3f8bfce7447ab604ec3639d1cb29 2024-12-09T01:42:12,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/c22365f5d6b046d68cbe8a160cd1a471 is 50, key is test_row_0/C:col10/1733708531646/Put/seqid=0 2024-12-09T01:42:12,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742143_1319 (size=12301) 2024-12-09T01:42:12,881 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=960 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/c22365f5d6b046d68cbe8a160cd1a471 2024-12-09T01:42:12,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:12,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708592870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:12,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:12,885 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:12,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708592871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:12,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:12,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:12,886 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:12,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/50f3f59d7c554f1c86283edae7767759 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/50f3f59d7c554f1c86283edae7767759 2024-12-09T01:42:12,893 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/50f3f59d7c554f1c86283edae7767759, entries=150, sequenceid=960, filesize=12.0 K 2024-12-09T01:42:12,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/30cb3f8bfce7447ab604ec3639d1cb29 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/30cb3f8bfce7447ab604ec3639d1cb29 2024-12-09T01:42:12,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:12,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708592883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:12,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:12,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708592885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:12,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:12,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708592886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:12,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:12,900 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/30cb3f8bfce7447ab604ec3639d1cb29, entries=150, sequenceid=960, filesize=12.0 K 2024-12-09T01:42:12,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/c22365f5d6b046d68cbe8a160cd1a471 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/c22365f5d6b046d68cbe8a160cd1a471 2024-12-09T01:42:12,907 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/c22365f5d6b046d68cbe8a160cd1a471, entries=150, sequenceid=960, filesize=12.0 K 2024-12-09T01:42:12,908 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for bb023e4ee9cc2de03187f3dff0a271ab in 231ms, sequenceid=960, compaction requested=false 2024-12-09T01:42:12,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:12,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:12,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-12-09T01:42:12,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-12-09T01:42:12,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-12-09T01:42:12,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 701 msec 2024-12-09T01:42:12,913 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 707 msec 2024-12-09T01:42:12,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:12,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=154.31 KB heapSize=459.66 KB 2024-12-09T01:42:12,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:12,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:12,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:12,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:12,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:12,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:13,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/6b56e89252eb4606934780dc42dc124e is 50, key is test_row_0/A:col10/1733708532878/Put/seqid=0 2024-12-09T01:42:13,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708593003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:13,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:13,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708593003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:13,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:13,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708593003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:13,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:13,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708593006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:13,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:13,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:13,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708593008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:13,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:13,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742144_1320 (size=14741) 2024-12-09T01:42:13,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:13,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708593124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:13,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:13,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708593124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:13,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:13,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:13,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:13,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708593214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:13,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:13,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,217 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708593214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:13,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708593214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:13,228 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/4e7adf136c88469ba3c188379da1b2d2 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/4e7adf136c88469ba3c188379da1b2d2 2024-12-09T01:42:13,234 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/C of bb023e4ee9cc2de03187f3dff0a271ab into 4e7adf136c88469ba3c188379da1b2d2(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:13,234 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:13,234 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/C, priority=12, startTime=1733708532635; duration=0sec 2024-12-09T01:42:13,234 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:13,234 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:C 2024-12-09T01:42:13,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:13,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708593333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:13,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:13,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708593333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:13,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:13,340 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:13,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-12-09T01:42:13,344 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:13,345 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:42:13,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-12-09T01:42:13,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T01:42:13,348 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:42:13,349 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:42:13,349 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:42:13,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=988 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/6b56e89252eb4606934780dc42dc124e 2024-12-09T01:42:13,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T01:42:13,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/8de5da2b8f1847caa88a400062a39305 is 50, key is test_row_0/B:col10/1733708532878/Put/seqid=0 2024-12-09T01:42:13,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742145_1321 (size=12301) 2024-12-09T01:42:13,499 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=988 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/8de5da2b8f1847caa88a400062a39305 2024-12-09T01:42:13,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T01:42:13,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:13,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:13,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:13,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:13,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:13,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:13,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/9949a1e9228b40ecb2d66240988b57a2 is 50, key is test_row_0/C:col10/1733708532878/Put/seqid=0 2024-12-09T01:42:13,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:13,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708593524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:13,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:13,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708593524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:13,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:13,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708593524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,532 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:13,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:13,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:13,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:13,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742146_1322 (size=12301) 2024-12-09T01:42:13,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=988 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/9949a1e9228b40ecb2d66240988b57a2 2024-12-09T01:42:13,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/6b56e89252eb4606934780dc42dc124e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/6b56e89252eb4606934780dc42dc124e 2024-12-09T01:42:13,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/6b56e89252eb4606934780dc42dc124e, entries=200, sequenceid=988, filesize=14.4 K 2024-12-09T01:42:13,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/8de5da2b8f1847caa88a400062a39305 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/8de5da2b8f1847caa88a400062a39305 2024-12-09T01:42:13,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/8de5da2b8f1847caa88a400062a39305, entries=150, sequenceid=988, filesize=12.0 K 2024-12-09T01:42:13,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/9949a1e9228b40ecb2d66240988b57a2 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/9949a1e9228b40ecb2d66240988b57a2 2024-12-09T01:42:13,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/9949a1e9228b40ecb2d66240988b57a2, entries=150, sequenceid=988, filesize=12.0 K 2024-12-09T01:42:13,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~459.61 KB/470640, currentSize=20.13 KB/20610 for bb023e4ee9cc2de03187f3dff0a271ab in 597ms, sequenceid=988, compaction requested=true 2024-12-09T01:42:13,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:13,593 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:13,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:42:13,596 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:13,596 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/A is initiating minor compaction (all files) 2024-12-09T01:42:13,596 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/A in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:13,596 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/865c842be51249a189b7053d91a8dbd8, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/50f3f59d7c554f1c86283edae7767759, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/6b56e89252eb4606934780dc42dc124e] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=39.1 K 2024-12-09T01:42:13,597 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:13,598 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:13,598 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 865c842be51249a189b7053d91a8dbd8, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=952, earliestPutTs=1733708530968 2024-12-09T01:42:13,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:42:13,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:13,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:42:13,599 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:13,599 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50f3f59d7c554f1c86283edae7767759, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=960, earliestPutTs=1733708531643 2024-12-09T01:42:13,601 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:13,602 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/B is initiating minor compaction (all files) 2024-12-09T01:42:13,602 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/B in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:13,602 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/72d8980527004649b33758faefd2bbb4, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/30cb3f8bfce7447ab604ec3639d1cb29, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/8de5da2b8f1847caa88a400062a39305] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=36.7 K 2024-12-09T01:42:13,602 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b56e89252eb4606934780dc42dc124e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=988, earliestPutTs=1733708532849 2024-12-09T01:42:13,602 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 72d8980527004649b33758faefd2bbb4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=952, earliestPutTs=1733708530968 2024-12-09T01:42:13,603 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 30cb3f8bfce7447ab604ec3639d1cb29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=960, earliestPutTs=1733708531643 2024-12-09T01:42:13,603 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 8de5da2b8f1847caa88a400062a39305, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=988, earliestPutTs=1733708532878 2024-12-09T01:42:13,637 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#A#compaction#279 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:13,638 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/8eeb4a3003314c19a2074a61c67fd26f is 50, key is test_row_0/A:col10/1733708532878/Put/seqid=0 2024-12-09T01:42:13,650 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#B#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:13,651 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/576e66b71a92455886288c0e9099e443 is 50, key is test_row_0/B:col10/1733708532878/Put/seqid=0 2024-12-09T01:42:13,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:42:13,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:13,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:13,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:13,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:13,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:13,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:13,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T01:42:13,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:13,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:13,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:13,660 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:13,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:13,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:13,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T01:42:13,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:13,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742147_1323 (size=13051) 2024-12-09T01:42:13,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/0fd0f2d572304324a33573930a51d5b3 is 50, key is test_row_0/A:col10/1733708533656/Put/seqid=0 2024-12-09T01:42:13,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742148_1324 (size=13051) 2024-12-09T01:42:13,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708533737, max timestamp=1733708533755Num uniques 859; 2024-12-09T01:42:13,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708533737, max timestamp=1733708533754Num uniques 859; 2024-12-09T01:42:13,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708533737, max timestamp=1733708533755Num uniques 859; 2024-12-09T01:42:13,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:42:13,760 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:42:13,761 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:42:13,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742149_1325 (size=14741) 2024-12-09T01:42:13,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=999 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/0fd0f2d572304324a33573930a51d5b3 2024-12-09T01:42:13,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/0b94c6663cdb4deabfaffb5118e09ddc is 50, key is test_row_0/B:col10/1733708533656/Put/seqid=0 2024-12-09T01:42:13,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T01:42:13,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:13,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:13,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:13,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:13,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:13,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:13,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742150_1326 (size=12301) 2024-12-09T01:42:13,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=999 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/0b94c6663cdb4deabfaffb5118e09ddc 2024-12-09T01:42:13,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/21b90410bfa94fbc9db44ef4d4ecf53f is 50, key is test_row_0/C:col10/1733708533656/Put/seqid=0 2024-12-09T01:42:13,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733708533796, max timestamp=1733708533871Num uniques 1018; 2024-12-09T01:42:13,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T01:42:13,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733708533796, max timestamp=1733708533871Num uniques 1018; 2024-12-09T01:42:13,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T01:42:13,883 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1733708533796, max timestamp=1733708533871Num uniques 1018; 2024-12-09T01:42:13,884 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-12-09T01:42:13,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742151_1327 (size=12301) 2024-12-09T01:42:13,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=999 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/21b90410bfa94fbc9db44ef4d4ecf53f 2024-12-09T01:42:13,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/0fd0f2d572304324a33573930a51d5b3 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/0fd0f2d572304324a33573930a51d5b3 2024-12-09T01:42:13,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/0fd0f2d572304324a33573930a51d5b3, entries=200, sequenceid=999, filesize=14.4 K 2024-12-09T01:42:13,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/0b94c6663cdb4deabfaffb5118e09ddc as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/0b94c6663cdb4deabfaffb5118e09ddc 2024-12-09T01:42:13,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/0b94c6663cdb4deabfaffb5118e09ddc, entries=150, sequenceid=999, filesize=12.0 K 2024-12-09T01:42:13,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/21b90410bfa94fbc9db44ef4d4ecf53f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/21b90410bfa94fbc9db44ef4d4ecf53f 2024-12-09T01:42:13,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/21b90410bfa94fbc9db44ef4d4ecf53f, entries=150, sequenceid=999, filesize=12.0 K 2024-12-09T01:42:13,939 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=100.63 KB/103050 for bb023e4ee9cc2de03187f3dff0a271ab in 281ms, sequenceid=999, compaction requested=true 2024-12-09T01:42:13,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:13,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:13,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:A, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:42:13,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T01:42:13,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:B, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:42:13,939 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-09T01:42:13,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:42:13,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-09T01:42:13,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=107.34 KB heapSize=299.77 KB 2024-12-09T01:42:13,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:13,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:13,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:13,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:13,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:13,941 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:13,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/1e8b04b735074a229ef2092333963108 is 50, key is test_row_0/A:col10/1733708533901/Put/seqid=0 2024-12-09T01:42:13,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T01:42:13,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:13,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:13,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:13,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:13,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T01:42:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:13,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742152_1328 (size=14741) 2024-12-09T01:42:14,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:14,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708594012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:14,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:14,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:14,025 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:14,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:14,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708594024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:14,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:14,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:14,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:14,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:14,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708594044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708594044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:14,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:14,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:14,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:14,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708594044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:14,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:14,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:14,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:14,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:14,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:14,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:14,111 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/8eeb4a3003314c19a2074a61c67fd26f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/8eeb4a3003314c19a2074a61c67fd26f 2024-12-09T01:42:14,116 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/A of bb023e4ee9cc2de03187f3dff0a271ab into 8eeb4a3003314c19a2074a61c67fd26f(size=12.7 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:14,116 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:14,116 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/A, priority=13, startTime=1733708533593; duration=0sec 2024-12-09T01:42:14,116 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-12-09T01:42:14,116 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:A 2024-12-09T01:42:14,116 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:A 2024-12-09T01:42:14,116 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:42:14,118 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:42:14,118 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/C is initiating minor compaction (all files) 2024-12-09T01:42:14,118 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/C in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:14,118 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/4e7adf136c88469ba3c188379da1b2d2, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/c22365f5d6b046d68cbe8a160cd1a471, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/9949a1e9228b40ecb2d66240988b57a2, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/21b90410bfa94fbc9db44ef4d4ecf53f] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=48.7 K 2024-12-09T01:42:14,119 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e7adf136c88469ba3c188379da1b2d2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=952, earliestPutTs=1733708530968 2024-12-09T01:42:14,119 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting c22365f5d6b046d68cbe8a160cd1a471, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=960, earliestPutTs=1733708531643 2024-12-09T01:42:14,120 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9949a1e9228b40ecb2d66240988b57a2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=988, earliestPutTs=1733708532878 2024-12-09T01:42:14,120 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21b90410bfa94fbc9db44ef4d4ecf53f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=999, earliestPutTs=1733708533004 2024-12-09T01:42:14,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T01:42:14,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:14,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:14,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:14,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:14,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:14,139 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#C#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:14,140 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/0d5c177769b24a8eba972f9bac780846 is 50, key is test_row_0/C:col10/1733708533656/Put/seqid=0 2024-12-09T01:42:14,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:14,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708594134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:14,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:14,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:14,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:14,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:14,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708594144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:14,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:14,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:14,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:14,155 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/576e66b71a92455886288c0e9099e443 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/576e66b71a92455886288c0e9099e443 2024-12-09T01:42:14,161 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/B of bb023e4ee9cc2de03187f3dff0a271ab into 576e66b71a92455886288c0e9099e443(size=12.7 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:14,161 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:14,161 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/B, priority=13, startTime=1733708533597; duration=0sec 2024-12-09T01:42:14,162 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-12-09T01:42:14,162 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:B 2024-12-09T01:42:14,162 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:B 2024-12-09T01:42:14,162 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-12-09T01:42:14,162 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T01:42:14,162 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T01:42:14,162 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. because compaction request was cancelled 2024-12-09T01:42:14,162 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:C 2024-12-09T01:42:14,163 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T01:42:14,164 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T01:42:14,164 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T01:42:14,164 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. because compaction request was cancelled 2024-12-09T01:42:14,164 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:B 2024-12-09T01:42:14,164 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T01:42:14,165 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T01:42:14,165 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T01:42:14,165 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. because compaction request was cancelled 2024-12-09T01:42:14,165 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:A 2024-12-09T01:42:14,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742153_1329 (size=13085) 2024-12-09T01:42:14,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T01:42:14,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:14,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:14,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:14,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:14,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:14,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:14,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708594354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:14,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:14,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:14,357 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:14,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:14,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708594363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:14,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:14,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:14,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:14,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=1048 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/1e8b04b735074a229ef2092333963108 2024-12-09T01:42:14,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/04946f33823c48fdb842af21dc864a15 is 50, key is test_row_0/B:col10/1733708533901/Put/seqid=0 2024-12-09T01:42:14,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742154_1330 (size=12301) 2024-12-09T01:42:14,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=1048 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/04946f33823c48fdb842af21dc864a15 2024-12-09T01:42:14,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T01:42:14,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:14,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:14,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:14,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:14,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:14,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/4cf99646c207415b96c90525d7b7e50f is 50, key is test_row_0/C:col10/1733708533901/Put/seqid=0 2024-12-09T01:42:14,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742155_1331 (size=12301) 2024-12-09T01:42:14,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T01:42:14,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T01:42:14,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:14,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:14,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:14,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:14,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:14,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:14,590 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/0d5c177769b24a8eba972f9bac780846 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/0d5c177769b24a8eba972f9bac780846 2024-12-09T01:42:14,597 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/C of bb023e4ee9cc2de03187f3dff0a271ab into 0d5c177769b24a8eba972f9bac780846(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:14,597 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:14,597 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/C, priority=12, startTime=1733708533939; duration=0sec 2024-12-09T01:42:14,597 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:14,598 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:C 2024-12-09T01:42:14,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:14,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708594663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:14,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:14,668 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:14,669 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:14,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:14,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 262 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708594673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:14,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:14,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:14,674 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:14,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T01:42:14,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:14,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:14,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:14,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:14,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:14,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=1048 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/4cf99646c207415b96c90525d7b7e50f 2024-12-09T01:42:14,854 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/1e8b04b735074a229ef2092333963108 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/1e8b04b735074a229ef2092333963108 2024-12-09T01:42:14,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/1e8b04b735074a229ef2092333963108, entries=200, sequenceid=1048, filesize=14.4 K 2024-12-09T01:42:14,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/04946f33823c48fdb842af21dc864a15 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/04946f33823c48fdb842af21dc864a15 2024-12-09T01:42:14,864 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/04946f33823c48fdb842af21dc864a15, entries=150, sequenceid=1048, filesize=12.0 K 2024-12-09T01:42:14,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/4cf99646c207415b96c90525d7b7e50f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/4cf99646c207415b96c90525d7b7e50f 2024-12-09T01:42:14,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/4cf99646c207415b96c90525d7b7e50f, entries=150, sequenceid=1048, filesize=12.0 K 2024-12-09T01:42:14,873 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~299.72 KB/306912, currentSize=73.80 KB/75570 for bb023e4ee9cc2de03187f3dff0a271ab in 933ms, sequenceid=1048, compaction requested=true 2024-12-09T01:42:14,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:14,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:42:14,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:14,874 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:14,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:42:14,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:14,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:42:14,874 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T01:42:14,874 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:14,875 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:14,875 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/B is initiating minor compaction (all files) 2024-12-09T01:42:14,875 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/B in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:14,875 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/576e66b71a92455886288c0e9099e443, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/0b94c6663cdb4deabfaffb5118e09ddc, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/04946f33823c48fdb842af21dc864a15] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=36.8 K 2024-12-09T01:42:14,875 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:14,875 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/A is initiating minor compaction (all files) 2024-12-09T01:42:14,875 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/A in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:14,875 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/8eeb4a3003314c19a2074a61c67fd26f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/0fd0f2d572304324a33573930a51d5b3, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/1e8b04b735074a229ef2092333963108] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=41.5 K 2024-12-09T01:42:14,876 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 576e66b71a92455886288c0e9099e443, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=988, earliestPutTs=1733708532878 2024-12-09T01:42:14,876 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8eeb4a3003314c19a2074a61c67fd26f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=988, earliestPutTs=1733708532878 2024-12-09T01:42:14,876 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b94c6663cdb4deabfaffb5118e09ddc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=999, earliestPutTs=1733708533004 2024-12-09T01:42:14,877 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fd0f2d572304324a33573930a51d5b3, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=999, earliestPutTs=1733708533004 2024-12-09T01:42:14,877 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 04946f33823c48fdb842af21dc864a15, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1048, earliestPutTs=1733708533901 2024-12-09T01:42:14,878 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e8b04b735074a229ef2092333963108, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=1048, earliestPutTs=1733708533901 2024-12-09T01:42:14,889 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#B#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:14,890 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/3701244c646142f299c9425850a911d0 is 50, key is test_row_0/B:col10/1733708533901/Put/seqid=0 2024-12-09T01:42:14,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-12-09T01:42:14,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:14,895 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T01:42:14,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:14,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:14,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:14,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:14,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:14,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:14,906 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#A#compaction#289 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:14,907 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/d0038cf0cf0945f0bcc077a757e345f8 is 50, key is test_row_0/A:col10/1733708533901/Put/seqid=0 2024-12-09T01:42:14,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/6495f3b4fc874fcaa4ec441a66c47c8d is 50, key is test_row_0/A:col10/1733708533961/Put/seqid=0 2024-12-09T01:42:14,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742156_1332 (size=13153) 2024-12-09T01:42:14,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742158_1334 (size=12301) 2024-12-09T01:42:14,929 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=1065 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/6495f3b4fc874fcaa4ec441a66c47c8d 2024-12-09T01:42:14,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742157_1333 (size=13153) 2024-12-09T01:42:14,942 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/d0038cf0cf0945f0bcc077a757e345f8 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/d0038cf0cf0945f0bcc077a757e345f8 2024-12-09T01:42:14,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/1c04f105f08b4c66b4cb1d82f9f4bef9 is 50, key is test_row_0/B:col10/1733708533961/Put/seqid=0 2024-12-09T01:42:14,950 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/A of bb023e4ee9cc2de03187f3dff0a271ab into d0038cf0cf0945f0bcc077a757e345f8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:14,950 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:14,950 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/A, priority=13, startTime=1733708534873; duration=0sec 2024-12-09T01:42:14,950 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:14,950 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:A 2024-12-09T01:42:14,950 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-09T01:42:14,952 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-12-09T01:42:14,952 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-12-09T01:42:14,952 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. because compaction request was cancelled 2024-12-09T01:42:14,952 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:C 2024-12-09T01:42:14,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742159_1335 (size=12301) 2024-12-09T01:42:14,959 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=1065 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/1c04f105f08b4c66b4cb1d82f9f4bef9 2024-12-09T01:42:14,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/effb8ecae8ac4dd5b818e99f988bb114 is 50, key is test_row_0/C:col10/1733708533961/Put/seqid=0 2024-12-09T01:42:15,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742160_1336 (size=12301) 2024-12-09T01:42:15,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:15,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708595095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:15,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:15,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708595097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:15,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:15,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 252 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708595100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:15,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:15,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708595173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:15,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:15,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708595184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:15,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:15,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708595203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:15,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:15,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708595213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,216 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:15,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708595213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,217 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:15,330 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/3701244c646142f299c9425850a911d0 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/3701244c646142f299c9425850a911d0 2024-12-09T01:42:15,337 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/B of bb023e4ee9cc2de03187f3dff0a271ab into 3701244c646142f299c9425850a911d0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:15,337 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:15,337 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/B, priority=13, startTime=1733708534874; duration=0sec 2024-12-09T01:42:15,337 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:15,337 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:B 2024-12-09T01:42:15,409 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=1065 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/effb8ecae8ac4dd5b818e99f988bb114 2024-12-09T01:42:15,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708595414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:15,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,417 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:15,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/6495f3b4fc874fcaa4ec441a66c47c8d as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/6495f3b4fc874fcaa4ec441a66c47c8d 2024-12-09T01:42:15,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708595424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,425 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:15,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708595427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:15,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:15,432 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/6495f3b4fc874fcaa4ec441a66c47c8d, entries=150, sequenceid=1065, filesize=12.0 K 2024-12-09T01:42:15,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/1c04f105f08b4c66b4cb1d82f9f4bef9 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/1c04f105f08b4c66b4cb1d82f9f4bef9 2024-12-09T01:42:15,440 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/1c04f105f08b4c66b4cb1d82f9f4bef9, entries=150, sequenceid=1065, filesize=12.0 K 2024-12-09T01:42:15,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/effb8ecae8ac4dd5b818e99f988bb114 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/effb8ecae8ac4dd5b818e99f988bb114 2024-12-09T01:42:15,447 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/effb8ecae8ac4dd5b818e99f988bb114, entries=150, sequenceid=1065, filesize=12.0 K 2024-12-09T01:42:15,449 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for bb023e4ee9cc2de03187f3dff0a271ab in 555ms, sequenceid=1065, compaction requested=true 2024-12-09T01:42:15,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:15,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:15,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-12-09T01:42:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-12-09T01:42:15,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-12-09T01:42:15,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1100 sec 2024-12-09T01:42:15,465 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.1180 sec 2024-12-09T01:42:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-12-09T01:42:15,503 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:15,504 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:42:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-12-09T01:42:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T01:42:15,506 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:42:15,506 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:42:15,507 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:42:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T01:42:15,659 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-12-09T01:42:15,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:15,660 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T01:42:15,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:15,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:15,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:15,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:15,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:15,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:15,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/75844ec575cc4d4c83d8290e646a790a is 50, key is test_row_0/A:col10/1733708535085/Put/seqid=0 2024-12-09T01:42:15,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742161_1337 (size=12301) 2024-12-09T01:42:15,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:15,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:15,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:15,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708595756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:15,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:15,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708595758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:15,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:15,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 227 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708595761, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,767 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:15,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T01:42:15,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:15,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708595864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:15,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:15,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:15,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708595873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:15,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708595873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:15,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:15,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:16,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:16,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708596073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:16,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:16,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:16,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708596083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:16,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:16,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708596083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:16,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:16,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:16,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:16,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:16,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:16,090 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:16,124 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=1085 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/75844ec575cc4d4c83d8290e646a790a 2024-12-09T01:42:16,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T01:42:16,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/14d3ccc7aaf24a47a4461985ebf383cc is 50, key is test_row_0/B:col10/1733708535085/Put/seqid=0 2024-12-09T01:42:16,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742162_1338 (size=12301) 2024-12-09T01:42:16,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:16,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708596193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:16,199 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_0' in bb023e4ee9cc2de03187f3dff0a271ab of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:42:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:16,199 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:16,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:16,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708596206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:16,209 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_0' in bb023e4ee9cc2de03187f3dff0a271ab of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:42:16,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:16,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:16,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:16,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:16,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708596384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:16,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:16,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:16,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708596393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:16,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:16,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:16,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708596393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:16,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:16,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:16,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:16,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:16,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:16,571 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=1085 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/14d3ccc7aaf24a47a4461985ebf383cc 2024-12-09T01:42:16,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/cf887e0ccabd4276b9f551daefb32a9a is 50, key is test_row_0/C:col10/1733708535085/Put/seqid=0 2024-12-09T01:42:16,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742163_1339 (size=12301) 2024-12-09T01:42:16,596 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=1085 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/cf887e0ccabd4276b9f551daefb32a9a 2024-12-09T01:42:16,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/75844ec575cc4d4c83d8290e646a790a as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/75844ec575cc4d4c83d8290e646a790a 2024-12-09T01:42:16,618 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/75844ec575cc4d4c83d8290e646a790a, entries=150, sequenceid=1085, filesize=12.0 K 2024-12-09T01:42:16,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/14d3ccc7aaf24a47a4461985ebf383cc as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/14d3ccc7aaf24a47a4461985ebf383cc 2024-12-09T01:42:16,631 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/14d3ccc7aaf24a47a4461985ebf383cc, entries=150, sequenceid=1085, filesize=12.0 K 2024-12-09T01:42:16,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/cf887e0ccabd4276b9f551daefb32a9a as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/cf887e0ccabd4276b9f551daefb32a9a 2024-12-09T01:42:16,640 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/cf887e0ccabd4276b9f551daefb32a9a, entries=150, sequenceid=1085, filesize=12.0 K 2024-12-09T01:42:16,643 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for bb023e4ee9cc2de03187f3dff0a271ab in 984ms, sequenceid=1085, compaction requested=true 2024-12-09T01:42:16,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:16,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:16,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-12-09T01:42:16,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T01:42:16,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-12-09T01:42:16,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-12-09T01:42:16,651 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1400 sec 2024-12-09T01:42:16,653 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.1480 sec 2024-12-09T01:42:16,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:16,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T01:42:16,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:16,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:16,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:16,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:16,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:16,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:16,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/8889fe7b6a6e40a9a6200ee160a07de4 is 50, key is test_row_0/A:col10/1733708536905/Put/seqid=0 2024-12-09T01:42:16,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742164_1340 (size=14741) 2024-12-09T01:42:16,940 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=1100 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/8889fe7b6a6e40a9a6200ee160a07de4 2024-12-09T01:42:16,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/3264ab6e544d46c8987123a3a1797afd is 50, key is test_row_0/B:col10/1733708536905/Put/seqid=0 2024-12-09T01:42:16,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:16,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708596954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:16,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:16,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:16,962 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:16,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708596962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:16,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:16,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:16,969 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:16,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:16,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708596962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:16,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:16,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:16,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:16,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742165_1341 (size=12301) 2024-12-09T01:42:17,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:17,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 219 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708597074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:17,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708597074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:17,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:17,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:17,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:17,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:17,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708597074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:17,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:17,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:17,081 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:17,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:17,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:17,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:17,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:17,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 220 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708597284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:17,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:17,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:17,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:17,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:17,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708597294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:17,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708597294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:17,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=1100 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/3264ab6e544d46c8987123a3a1797afd 2024-12-09T01:42:17,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/a4a88cc38d23465fa7a49995c9de9339 is 50, key is test_row_0/C:col10/1733708536905/Put/seqid=0 2024-12-09T01:42:17,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742166_1342 (size=12301) 2024-12-09T01:42:17,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:17,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708597593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:17,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:17,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:17,598 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:17,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:17,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708597604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:17,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:17,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708597604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:17,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:17,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:17,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:17,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:17,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:17,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:17,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-12-09T01:42:17,654 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:17,654 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:42:17,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-12-09T01:42:17,656 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:42:17,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-09T01:42:17,657 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:42:17,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:42:17,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-09T01:42:17,805 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=1100 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/a4a88cc38d23465fa7a49995c9de9339 2024-12-09T01:42:17,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-09T01:42:17,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:17,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:17,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:17,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:17,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:17,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:17,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/8889fe7b6a6e40a9a6200ee160a07de4 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/8889fe7b6a6e40a9a6200ee160a07de4 2024-12-09T01:42:17,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/8889fe7b6a6e40a9a6200ee160a07de4, entries=200, sequenceid=1100, filesize=14.4 K 2024-12-09T01:42:17,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/3264ab6e544d46c8987123a3a1797afd as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/3264ab6e544d46c8987123a3a1797afd 2024-12-09T01:42:17,823 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/3264ab6e544d46c8987123a3a1797afd, entries=150, sequenceid=1100, filesize=12.0 K 2024-12-09T01:42:17,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/a4a88cc38d23465fa7a49995c9de9339 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/a4a88cc38d23465fa7a49995c9de9339 2024-12-09T01:42:17,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/a4a88cc38d23465fa7a49995c9de9339, entries=150, sequenceid=1100, filesize=12.0 K 2024-12-09T01:42:17,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for bb023e4ee9cc2de03187f3dff0a271ab in 922ms, sequenceid=1100, compaction requested=true 2024-12-09T01:42:17,829 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:17,829 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:42:17,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:42:17,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:17,830 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:42:17,831 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52496 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:42:17,831 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/A is initiating minor compaction (all files) 2024-12-09T01:42:17,831 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/A in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:17,831 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/d0038cf0cf0945f0bcc077a757e345f8, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/6495f3b4fc874fcaa4ec441a66c47c8d, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/75844ec575cc4d4c83d8290e646a790a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/8889fe7b6a6e40a9a6200ee160a07de4] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=51.3 K 2024-12-09T01:42:17,832 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50056 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:42:17,832 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/B is initiating minor compaction (all files) 2024-12-09T01:42:17,832 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/B in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:17,832 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/3701244c646142f299c9425850a911d0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/1c04f105f08b4c66b4cb1d82f9f4bef9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/14d3ccc7aaf24a47a4461985ebf383cc, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/3264ab6e544d46c8987123a3a1797afd] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=48.9 K 2024-12-09T01:42:17,833 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting d0038cf0cf0945f0bcc077a757e345f8, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=1048, earliestPutTs=1733708533901 2024-12-09T01:42:17,833 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3701244c646142f299c9425850a911d0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=1048, earliestPutTs=1733708533901 2024-12-09T01:42:17,832 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:42:17,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:17,833 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c04f105f08b4c66b4cb1d82f9f4bef9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1065, earliestPutTs=1733708533961 2024-12-09T01:42:17,833 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 6495f3b4fc874fcaa4ec441a66c47c8d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1065, earliestPutTs=1733708533961 2024-12-09T01:42:17,834 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14d3ccc7aaf24a47a4461985ebf383cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1085, earliestPutTs=1733708535085 2024-12-09T01:42:17,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:42:17,834 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 75844ec575cc4d4c83d8290e646a790a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1085, earliestPutTs=1733708535085 2024-12-09T01:42:17,834 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3264ab6e544d46c8987123a3a1797afd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1100, earliestPutTs=1733708535743 2024-12-09T01:42:17,834 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 8889fe7b6a6e40a9a6200ee160a07de4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=1100, earliestPutTs=1733708535743 2024-12-09T01:42:17,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:17,855 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#A#compaction#299 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:17,856 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/2a71b0c9e0444db48814117c4b0f8531 is 50, key is test_row_0/A:col10/1733708536905/Put/seqid=0 2024-12-09T01:42:17,860 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#B#compaction#300 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:17,861 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/8fd12999c354424481ac707d8e9c6748 is 50, key is test_row_0/B:col10/1733708536905/Put/seqid=0 2024-12-09T01:42:17,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742168_1344 (size=13289) 2024-12-09T01:42:17,888 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/8fd12999c354424481ac707d8e9c6748 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/8fd12999c354424481ac707d8e9c6748 2024-12-09T01:42:17,894 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/B of bb023e4ee9cc2de03187f3dff0a271ab into 8fd12999c354424481ac707d8e9c6748(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:17,894 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:17,894 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/B, priority=12, startTime=1733708537830; duration=0sec 2024-12-09T01:42:17,894 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:17,894 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:B 2024-12-09T01:42:17,895 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T01:42:17,896 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62289 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T01:42:17,896 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/C is initiating minor compaction (all files) 2024-12-09T01:42:17,896 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/C in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:17,896 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/0d5c177769b24a8eba972f9bac780846, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/4cf99646c207415b96c90525d7b7e50f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/effb8ecae8ac4dd5b818e99f988bb114, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/cf887e0ccabd4276b9f551daefb32a9a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/a4a88cc38d23465fa7a49995c9de9339] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=60.8 K 2024-12-09T01:42:17,897 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d5c177769b24a8eba972f9bac780846, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=999, earliestPutTs=1733708533004 2024-12-09T01:42:17,897 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cf99646c207415b96c90525d7b7e50f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1048, earliestPutTs=1733708533901 2024-12-09T01:42:17,897 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting effb8ecae8ac4dd5b818e99f988bb114, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1065, earliestPutTs=1733708533961 2024-12-09T01:42:17,898 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf887e0ccabd4276b9f551daefb32a9a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1085, earliestPutTs=1733708535085 2024-12-09T01:42:17,898 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4a88cc38d23465fa7a49995c9de9339, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1100, earliestPutTs=1733708535743 2024-12-09T01:42:17,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742167_1343 (size=13289) 2024-12-09T01:42:17,910 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#C#compaction#301 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:17,910 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/5dd61c215d6c4be0aac16bd6abe706d6 is 50, key is test_row_0/C:col10/1733708536905/Put/seqid=0 2024-12-09T01:42:17,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742169_1345 (size=13255) 2024-12-09T01:42:17,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-12-09T01:42:17,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:17,965 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T01:42:17,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:17,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:17,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:17,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:17,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:17,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:17,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/1ff26ac0c9424be6ac4605aab0bb777f is 50, key is test_row_0/A:col10/1733708536960/Put/seqid=0 2024-12-09T01:42:17,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-09T01:42:17,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742170_1346 (size=12301) 2024-12-09T01:42:17,997 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=1119 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/1ff26ac0c9424be6ac4605aab0bb777f 2024-12-09T01:42:18,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/7fee44383491414ab4d7398a44c6edee is 50, key is test_row_0/B:col10/1733708536960/Put/seqid=0 2024-12-09T01:42:18,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742171_1347 (size=12301) 2024-12-09T01:42:18,040 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=1119 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/7fee44383491414ab4d7398a44c6edee 2024-12-09T01:42:18,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/8c4055e0d5624b019d9c3d3446a05dfd is 50, key is test_row_0/C:col10/1733708536960/Put/seqid=0 2024-12-09T01:42:18,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742172_1348 (size=12301) 2024-12-09T01:42:18,076 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=1119 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/8c4055e0d5624b019d9c3d3446a05dfd 2024-12-09T01:42:18,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/1ff26ac0c9424be6ac4605aab0bb777f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/1ff26ac0c9424be6ac4605aab0bb777f 2024-12-09T01:42:18,093 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/1ff26ac0c9424be6ac4605aab0bb777f, entries=150, sequenceid=1119, filesize=12.0 K 2024-12-09T01:42:18,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/7fee44383491414ab4d7398a44c6edee as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/7fee44383491414ab4d7398a44c6edee 2024-12-09T01:42:18,101 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/7fee44383491414ab4d7398a44c6edee, entries=150, sequenceid=1119, filesize=12.0 K 2024-12-09T01:42:18,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/8c4055e0d5624b019d9c3d3446a05dfd as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/8c4055e0d5624b019d9c3d3446a05dfd 2024-12-09T01:42:18,108 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/8c4055e0d5624b019d9c3d3446a05dfd, entries=150, sequenceid=1119, filesize=12.0 K 2024-12-09T01:42:18,109 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for bb023e4ee9cc2de03187f3dff0a271ab in 144ms, sequenceid=1119, compaction requested=false 2024-12-09T01:42:18,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:18,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:18,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-12-09T01:42:18,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-12-09T01:42:18,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-12-09T01:42:18,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 453 msec 2024-12-09T01:42:18,121 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 465 msec 2024-12-09T01:42:18,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:18,129 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:42:18,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:18,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:18,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:18,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:18,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:18,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:18,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/4897339f8d7544ce89f618595c06e3ef is 50, key is test_row_0/A:col10/1733708538127/Put/seqid=0 2024-12-09T01:42:18,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742173_1349 (size=14741) 2024-12-09T01:42:18,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=1130 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/4897339f8d7544ce89f618595c06e3ef 2024-12-09T01:42:18,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/71da7995948f4f83aace2f1169847167 is 50, key is test_row_0/B:col10/1733708538127/Put/seqid=0 2024-12-09T01:42:18,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:18,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708598183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:18,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:18,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 248 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708598191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:18,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708538172, max timestamp=1733708538192Num uniques 859; 2024-12-09T01:42:18,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708538172, max timestamp=1733708538192Num uniques 859; 2024-12-09T01:42:18,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708538172, max timestamp=1733708538192Num uniques 859; 2024-12-09T01:42:18,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:42:18,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:42:18,198 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:42:18,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:18,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:18,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742174_1350 (size=12301) 2024-12-09T01:42:18,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:18,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708598253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:18,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,263 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:18,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 288 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708598260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:18,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:18,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:18,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708598262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:18,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,273 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:18,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-12-09T01:42:18,284 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:18,285 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:42:18,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-12-09T01:42:18,287 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:42:18,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-09T01:42:18,288 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:42:18,288 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:42:18,306 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/2a71b0c9e0444db48814117c4b0f8531 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/2a71b0c9e0444db48814117c4b0f8531 2024-12-09T01:42:18,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:18,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 249 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708598303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:18,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,312 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:18,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708598303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:18,312 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/A of bb023e4ee9cc2de03187f3dff0a271ab into 2a71b0c9e0444db48814117c4b0f8531(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:18,312 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:18,312 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/A, priority=12, startTime=1733708537829; duration=0sec 2024-12-09T01:42:18,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,312 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:18,312 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:18,312 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:A 2024-12-09T01:42:18,335 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/5dd61c215d6c4be0aac16bd6abe706d6 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/5dd61c215d6c4be0aac16bd6abe706d6 2024-12-09T01:42:18,348 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/C of bb023e4ee9cc2de03187f3dff0a271ab into 5dd61c215d6c4be0aac16bd6abe706d6(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:18,348 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:18,348 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/C, priority=11, startTime=1733708537833; duration=0sec 2024-12-09T01:42:18,348 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:18,349 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:C 2024-12-09T01:42:18,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:18,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708598374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:18,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:18,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 289 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708598374, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,375 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:18,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:18,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708598383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:18,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,389 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:18,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-09T01:42:18,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-09T01:42:18,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:18,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:18,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:18,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:18,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:18,519 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:18,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 250 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708598513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:18,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:18,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708598524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:18,528 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:18,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 290 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708598583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708598583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:18,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-09T01:42:18,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:18,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:18,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:18,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:18,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:18,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:18,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:18,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 273 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708598594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:18,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,599 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:18,600 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=1130 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/71da7995948f4f83aace2f1169847167 2024-12-09T01:42:18,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-09T01:42:18,610 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/b556ea2c768c449a8d9c039d01b39fb2 is 50, key is test_row_0/C:col10/1733708538127/Put/seqid=0 2024-12-09T01:42:18,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742175_1351 (size=12301) 2024-12-09T01:42:18,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=1130 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/b556ea2c768c449a8d9c039d01b39fb2 2024-12-09T01:42:18,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/4897339f8d7544ce89f618595c06e3ef as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/4897339f8d7544ce89f618595c06e3ef 2024-12-09T01:42:18,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/4897339f8d7544ce89f618595c06e3ef, entries=200, sequenceid=1130, filesize=14.4 K 2024-12-09T01:42:18,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/71da7995948f4f83aace2f1169847167 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/71da7995948f4f83aace2f1169847167 2024-12-09T01:42:18,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/71da7995948f4f83aace2f1169847167, entries=150, sequenceid=1130, filesize=12.0 K 2024-12-09T01:42:18,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/b556ea2c768c449a8d9c039d01b39fb2 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/b556ea2c768c449a8d9c039d01b39fb2 2024-12-09T01:42:18,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/b556ea2c768c449a8d9c039d01b39fb2, entries=150, sequenceid=1130, filesize=12.0 K 2024-12-09T01:42:18,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for bb023e4ee9cc2de03187f3dff0a271ab in 524ms, sequenceid=1130, compaction requested=true 2024-12-09T01:42:18,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:18,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:42:18,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:18,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:42:18,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:18,653 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:18,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:42:18,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:18,653 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:18,654 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:18,654 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:18,654 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/A is initiating minor compaction (all files) 2024-12-09T01:42:18,654 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/B is initiating minor compaction (all files) 2024-12-09T01:42:18,654 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/A in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:18,654 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/B in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:18,654 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/2a71b0c9e0444db48814117c4b0f8531, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/1ff26ac0c9424be6ac4605aab0bb777f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/4897339f8d7544ce89f618595c06e3ef] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=39.4 K 2024-12-09T01:42:18,655 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/8fd12999c354424481ac707d8e9c6748, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/7fee44383491414ab4d7398a44c6edee, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/71da7995948f4f83aace2f1169847167] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=37.0 K 2024-12-09T01:42:18,655 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fd12999c354424481ac707d8e9c6748, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=1100, earliestPutTs=1733708535743 2024-12-09T01:42:18,655 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a71b0c9e0444db48814117c4b0f8531, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=1100, earliestPutTs=1733708535743 2024-12-09T01:42:18,655 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fee44383491414ab4d7398a44c6edee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1119, earliestPutTs=1733708536951 2024-12-09T01:42:18,655 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ff26ac0c9424be6ac4605aab0bb777f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1119, earliestPutTs=1733708536951 2024-12-09T01:42:18,656 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 71da7995948f4f83aace2f1169847167, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1130, earliestPutTs=1733708538120 2024-12-09T01:42:18,656 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4897339f8d7544ce89f618595c06e3ef, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=1130, earliestPutTs=1733708538120 2024-12-09T01:42:18,680 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#A#compaction#308 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:18,681 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/6e7dd021493c4c41845c1ecd138310af is 50, key is test_row_0/A:col10/1733708538127/Put/seqid=0 2024-12-09T01:42:18,682 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#B#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:18,683 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/074759a1fb45473a9ee76493471ab6f8 is 50, key is test_row_0/B:col10/1733708538127/Put/seqid=0 2024-12-09T01:42:18,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742176_1352 (size=13391) 2024-12-09T01:42:18,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742177_1353 (size=13391) 2024-12-09T01:42:18,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-12-09T01:42:18,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:18,750 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-12-09T01:42:18,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:18,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:18,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:18,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:18,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:18,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:18,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/b27424cebe744f49bf49bd6b52e81dea is 50, key is test_row_0/A:col10/1733708538257/Put/seqid=0 2024-12-09T01:42:18,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742178_1354 (size=12301) 2024-12-09T01:42:18,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:18,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708598855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:18,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:18,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708598856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:18,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:18,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 291 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708598894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708598894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:18,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:18,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708598904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:18,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,909 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-09T01:42:18,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708598974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:18,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:18,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,979 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708598974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:18,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:18,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:19,123 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/6e7dd021493c4c41845c1ecd138310af as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/6e7dd021493c4c41845c1ecd138310af 2024-12-09T01:42:19,130 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/A of bb023e4ee9cc2de03187f3dff0a271ab into 6e7dd021493c4c41845c1ecd138310af(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:19,131 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:19,131 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/A, priority=13, startTime=1733708538653; duration=0sec 2024-12-09T01:42:19,131 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:19,131 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:A 2024-12-09T01:42:19,131 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:19,132 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:19,132 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/C is initiating minor compaction (all files) 2024-12-09T01:42:19,132 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/C in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:19,132 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/5dd61c215d6c4be0aac16bd6abe706d6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/8c4055e0d5624b019d9c3d3446a05dfd, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/b556ea2c768c449a8d9c039d01b39fb2] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=37.0 K 2024-12-09T01:42:19,132 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5dd61c215d6c4be0aac16bd6abe706d6, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=1100, earliestPutTs=1733708535743 2024-12-09T01:42:19,133 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c4055e0d5624b019d9c3d3446a05dfd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1119, earliestPutTs=1733708536951 2024-12-09T01:42:19,133 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting b556ea2c768c449a8d9c039d01b39fb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1130, earliestPutTs=1733708538120 2024-12-09T01:42:19,137 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/074759a1fb45473a9ee76493471ab6f8 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/074759a1fb45473a9ee76493471ab6f8 2024-12-09T01:42:19,142 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#C#compaction#311 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:19,142 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/71bcb1db2d7844759320cb9a6ce87f8e is 50, key is test_row_0/C:col10/1733708538127/Put/seqid=0 2024-12-09T01:42:19,144 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/B of bb023e4ee9cc2de03187f3dff0a271ab into 074759a1fb45473a9ee76493471ab6f8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:19,144 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:19,144 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/B, priority=13, startTime=1733708538653; duration=0sec 2024-12-09T01:42:19,144 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:19,144 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:B 2024-12-09T01:42:19,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742179_1355 (size=13357) 2024-12-09T01:42:19,159 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/71bcb1db2d7844759320cb9a6ce87f8e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/71bcb1db2d7844759320cb9a6ce87f8e 2024-12-09T01:42:19,169 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/C of bb023e4ee9cc2de03187f3dff0a271ab into 71bcb1db2d7844759320cb9a6ce87f8e(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:19,169 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:19,169 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/C, priority=13, startTime=1733708538653; duration=0sec 2024-12-09T01:42:19,169 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:19,169 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:C 2024-12-09T01:42:19,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708599184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:19,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708599184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:19,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:19,195 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=1167 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/b27424cebe744f49bf49bd6b52e81dea 2024-12-09T01:42:19,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/7a3be0aea9984e478ef4f95e9baa6f91 is 50, key is test_row_0/B:col10/1733708538257/Put/seqid=0 2024-12-09T01:42:19,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742180_1356 (size=12301) 2024-12-09T01:42:19,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:19,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 272 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708599404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:19,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:19,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:19,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:19,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:19,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 292 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708599413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:19,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:19,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708599413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:19,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:19,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-09T01:42:19,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:19,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708599493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:19,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:19,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:19,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:19,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 259 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708599493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:19,499 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:19,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:19,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:19,499 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:19,624 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=1167 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/7a3be0aea9984e478ef4f95e9baa6f91 2024-12-09T01:42:19,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/a6139ae70d3845a089ae003e6e1f5733 is 50, key is test_row_0/C:col10/1733708538257/Put/seqid=0 2024-12-09T01:42:19,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742181_1357 (size=12301) 2024-12-09T01:42:20,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:20,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 241 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708600004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:20,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:20,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 260 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708600004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:20,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:20,066 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=1167 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/a6139ae70d3845a089ae003e6e1f5733 2024-12-09T01:42:20,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/b27424cebe744f49bf49bd6b52e81dea as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/b27424cebe744f49bf49bd6b52e81dea 2024-12-09T01:42:20,079 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/b27424cebe744f49bf49bd6b52e81dea, entries=150, sequenceid=1167, filesize=12.0 K 2024-12-09T01:42:20,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/7a3be0aea9984e478ef4f95e9baa6f91 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/7a3be0aea9984e478ef4f95e9baa6f91 2024-12-09T01:42:20,085 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/7a3be0aea9984e478ef4f95e9baa6f91, entries=150, sequenceid=1167, filesize=12.0 K 2024-12-09T01:42:20,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/a6139ae70d3845a089ae003e6e1f5733 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/a6139ae70d3845a089ae003e6e1f5733 2024-12-09T01:42:20,100 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/a6139ae70d3845a089ae003e6e1f5733, entries=150, sequenceid=1167, filesize=12.0 K 2024-12-09T01:42:20,102 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for bb023e4ee9cc2de03187f3dff0a271ab in 1352ms, sequenceid=1167, compaction requested=false 2024-12-09T01:42:20,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:20,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:20,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-12-09T01:42:20,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-12-09T01:42:20,106 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-12-09T01:42:20,106 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8160 sec 2024-12-09T01:42:20,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.8220 sec 2024-12-09T01:42:20,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:20,418 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T01:42:20,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:20,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:20,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:20,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:20,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:20,418 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:20,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/90cd24aa68334738b30a8a5745f37456 is 50, key is test_row_0/A:col10/1733708540415/Put/seqid=0 2024-12-09T01:42:20,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-12-09T01:42:20,436 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:20,437 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:42:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-12-09T01:42:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-09T01:42:20,439 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:42:20,440 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:42:20,440 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:42:20,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742182_1358 (size=14741) 2024-12-09T01:42:20,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:20,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 297 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708600477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:20,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:20,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:20,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:20,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:20,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708600479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:20,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:20,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:20,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:20,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:20,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708600486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:20,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-09T01:42:20,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-09T01:42:20,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:20,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:20,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:20,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:20,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:20,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 298 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708600593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:20,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708600593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:20,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:20,597 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:20,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708600593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:20,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:20,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:20,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:20,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:20,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:20,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:20,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-09T01:42:20,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:20,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:20,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:20,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:20,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:20,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:20,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-09T01:42:20,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:20,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708600803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:20,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:20,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:20,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708600804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:20,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:20,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:20,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:20,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708600804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:20,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:20,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:20,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:20,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:20,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:20,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:20,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=1182 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/90cd24aa68334738b30a8a5745f37456 2024-12-09T01:42:20,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/4a55f5b0a0cf407082070d281e76c00c is 50, key is test_row_0/B:col10/1733708540415/Put/seqid=0 2024-12-09T01:42:20,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-09T01:42:20,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:20,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:20,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:20,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:20,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:20,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:20,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742183_1359 (size=12301) 2024-12-09T01:42:21,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 261 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39890 deadline: 1733708601014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:21,017 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_1' in bb023e4ee9cc2de03187f3dff0a271ab of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:42:21,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:21,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:21,018 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:21,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:21,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 242 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39832 deadline: 1733708601023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:21,026 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_1' in bb023e4ee9cc2de03187f3dff0a271ab of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:42:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:21,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:21,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-09T01:42:21,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:21,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:21,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:21,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:21,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:21,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:21,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-09T01:42:21,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708601115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:21,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 284 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708601115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:21,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708601116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:21,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-09T01:42:21,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:21,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:21,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:21,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:21,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:21,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:21,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=1182 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/4a55f5b0a0cf407082070d281e76c00c 2024-12-09T01:42:21,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/ca4dc3dc68a542e7bae766c406a076ae is 50, key is test_row_0/C:col10/1733708540415/Put/seqid=0 2024-12-09T01:42:21,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742184_1360 (size=12301) 2024-12-09T01:42:21,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-09T01:42:21,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:21,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:21,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:21,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:21,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:21,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:21,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-09T01:42:21,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:21,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:21,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:21,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:21,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:21,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:21,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-09T01:42:21,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39842 deadline: 1733708601624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:21,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:21,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 286 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39818 deadline: 1733708601633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:21,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:21,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:21,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:21,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 285 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39850 deadline: 1733708601633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:21,636 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:21,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-09T01:42:21,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:21,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:21,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:21,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:21,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:21,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:21,720 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=1182 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/ca4dc3dc68a542e7bae766c406a076ae 2024-12-09T01:42:21,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/90cd24aa68334738b30a8a5745f37456 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/90cd24aa68334738b30a8a5745f37456 2024-12-09T01:42:21,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/90cd24aa68334738b30a8a5745f37456, entries=200, sequenceid=1182, filesize=14.4 K 2024-12-09T01:42:21,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/4a55f5b0a0cf407082070d281e76c00c as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/4a55f5b0a0cf407082070d281e76c00c 2024-12-09T01:42:21,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/4a55f5b0a0cf407082070d281e76c00c, entries=150, sequenceid=1182, filesize=12.0 K 2024-12-09T01:42:21,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/ca4dc3dc68a542e7bae766c406a076ae as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/ca4dc3dc68a542e7bae766c406a076ae 2024-12-09T01:42:21,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/ca4dc3dc68a542e7bae766c406a076ae, entries=150, sequenceid=1182, filesize=12.0 K 2024-12-09T01:42:21,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for bb023e4ee9cc2de03187f3dff0a271ab in 1327ms, sequenceid=1182, compaction requested=true 2024-12-09T01:42:21,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:21,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:42:21,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:21,745 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:21,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:42:21,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:21,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store bb023e4ee9cc2de03187f3dff0a271ab:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:42:21,745 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:21,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:21,746 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:21,746 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/B is initiating minor compaction (all files) 2024-12-09T01:42:21,747 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/B in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:21,747 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/074759a1fb45473a9ee76493471ab6f8, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/7a3be0aea9984e478ef4f95e9baa6f91, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/4a55f5b0a0cf407082070d281e76c00c] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=37.1 K 2024-12-09T01:42:21,747 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:21,747 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/A is initiating minor compaction (all files) 2024-12-09T01:42:21,747 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/A in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:21,747 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/6e7dd021493c4c41845c1ecd138310af, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/b27424cebe744f49bf49bd6b52e81dea, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/90cd24aa68334738b30a8a5745f37456] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=39.5 K 2024-12-09T01:42:21,747 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 074759a1fb45473a9ee76493471ab6f8, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=1130, earliestPutTs=1733708538120 2024-12-09T01:42:21,747 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e7dd021493c4c41845c1ecd138310af, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=1130, earliestPutTs=1733708538120 2024-12-09T01:42:21,748 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a3be0aea9984e478ef4f95e9baa6f91, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1167, earliestPutTs=1733708538251 2024-12-09T01:42:21,748 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting b27424cebe744f49bf49bd6b52e81dea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1167, earliestPutTs=1733708538251 2024-12-09T01:42:21,748 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90cd24aa68334738b30a8a5745f37456, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=1182, earliestPutTs=1733708538846 2024-12-09T01:42:21,748 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a55f5b0a0cf407082070d281e76c00c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1182, earliestPutTs=1733708538848 2024-12-09T01:42:21,757 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#A#compaction#317 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:21,758 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/34a9a853bf1c428180e2c244ada073e0 is 50, key is test_row_0/A:col10/1733708540415/Put/seqid=0 2024-12-09T01:42:21,777 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#B#compaction#318 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:21,777 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/6169009de0c1495eaf131b995b43db67 is 50, key is test_row_0/B:col10/1733708540415/Put/seqid=0 2024-12-09T01:42:21,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742185_1361 (size=13493) 2024-12-09T01:42:21,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742186_1362 (size=13493) 2024-12-09T01:42:21,797 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/6169009de0c1495eaf131b995b43db67 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/6169009de0c1495eaf131b995b43db67 2024-12-09T01:42:21,805 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/B of bb023e4ee9cc2de03187f3dff0a271ab into 6169009de0c1495eaf131b995b43db67(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:21,805 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:21,805 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/B, priority=13, startTime=1733708541745; duration=0sec 2024-12-09T01:42:21,805 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:21,805 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:B 2024-12-09T01:42:21,805 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:21,807 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:21,807 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): bb023e4ee9cc2de03187f3dff0a271ab/C is initiating minor compaction (all files) 2024-12-09T01:42:21,807 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of bb023e4ee9cc2de03187f3dff0a271ab/C in TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:21,808 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/71bcb1db2d7844759320cb9a6ce87f8e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/a6139ae70d3845a089ae003e6e1f5733, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/ca4dc3dc68a542e7bae766c406a076ae] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp, totalSize=37.1 K 2024-12-09T01:42:21,808 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 71bcb1db2d7844759320cb9a6ce87f8e, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=1130, earliestPutTs=1733708538120 2024-12-09T01:42:21,808 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting a6139ae70d3845a089ae003e6e1f5733, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1167, earliestPutTs=1733708538251 2024-12-09T01:42:21,809 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting ca4dc3dc68a542e7bae766c406a076ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1182, earliestPutTs=1733708538848 2024-12-09T01:42:21,820 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): bb023e4ee9cc2de03187f3dff0a271ab#C#compaction#319 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:21,820 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/45fd21d487594ef5a83989f363447fe8 is 50, key is test_row_0/C:col10/1733708540415/Put/seqid=0 2024-12-09T01:42:21,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-12-09T01:42:21,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:21,826 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T01:42:21,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:21,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:21,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:21,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:21,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:21,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:21,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/1b83f544555045aba75289310f052f6e is 50, key is test_row_0/A:col10/1733708540474/Put/seqid=0 2024-12-09T01:42:21,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742187_1363 (size=13459) 2024-12-09T01:42:21,877 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/45fd21d487594ef5a83989f363447fe8 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/45fd21d487594ef5a83989f363447fe8 2024-12-09T01:42:21,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742188_1364 (size=12301) 2024-12-09T01:42:21,885 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/C of bb023e4ee9cc2de03187f3dff0a271ab into 45fd21d487594ef5a83989f363447fe8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:21,885 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=1203 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/1b83f544555045aba75289310f052f6e 2024-12-09T01:42:21,885 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:21,885 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/C, priority=13, startTime=1733708541745; duration=0sec 2024-12-09T01:42:21,885 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:21,885 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:C 2024-12-09T01:42:21,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/541dfd7f37574d11b03e25dccc6a7854 is 50, key is test_row_0/B:col10/1733708540474/Put/seqid=0 2024-12-09T01:42:21,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742189_1365 (size=12301) 2024-12-09T01:42:21,959 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T01:42:22,188 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/34a9a853bf1c428180e2c244ada073e0 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/34a9a853bf1c428180e2c244ada073e0 2024-12-09T01:42:22,193 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in bb023e4ee9cc2de03187f3dff0a271ab/A of bb023e4ee9cc2de03187f3dff0a271ab into 34a9a853bf1c428180e2c244ada073e0(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:22,193 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:22,193 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., storeName=bb023e4ee9cc2de03187f3dff0a271ab/A, priority=13, startTime=1733708541745; duration=0sec 2024-12-09T01:42:22,193 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:22,193 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: bb023e4ee9cc2de03187f3dff0a271ab:A 2024-12-09T01:42:22,331 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=1203 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/541dfd7f37574d11b03e25dccc6a7854 2024-12-09T01:42:22,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/c4d2423e06944945bd93c4424d137fc1 is 50, key is test_row_0/C:col10/1733708540474/Put/seqid=0 2024-12-09T01:42:22,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742190_1366 (size=12301) 2024-12-09T01:42:22,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-09T01:42:22,588 INFO [Thread-1200 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1200. 2024-12-09T01:42:22,588 DEBUG [Thread-1200 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:42:22,588 DEBUG [Thread-1200 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:22,588 DEBUG [Thread-1200 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:22,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:42:22,589 INFO [Thread-1202 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1202. 2024-12-09T01:42:22,589 DEBUG [Thread-1202 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:42:22,589 DEBUG [Thread-1202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:22,589 DEBUG [Thread-1202 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:22,589 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:42:22,590 INFO [Thread-1196 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1196. 2024-12-09T01:42:22,590 DEBUG [Thread-1196 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:42:22,590 DEBUG [Thread-1196 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:22,590 DEBUG [Thread-1196 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:22,590 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:42:22,591 INFO [Thread-1204 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1204. 2024-12-09T01:42:22,591 DEBUG [Thread-1204 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:42:22,591 DEBUG [Thread-1204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:22,591 DEBUG [Thread-1204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:22,591 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:42:22,591 INFO [Thread-1198 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1198. 2024-12-09T01:42:22,591 DEBUG [Thread-1198 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:42:22,591 DEBUG [Thread-1198 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:22,591 DEBUG [Thread-1198 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:22,591 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:42:22,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:22,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. as already flushing 2024-12-09T01:42:22,645 INFO [Thread-1191 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1191. 2024-12-09T01:42:22,645 INFO [Thread-1193 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1193. 2024-12-09T01:42:22,645 DEBUG [Thread-1191 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:42:22,645 DEBUG [Thread-1193 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:42:22,645 DEBUG [Thread-1191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:22,645 DEBUG [Thread-1193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:22,645 DEBUG [Thread-1193 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:22,645 DEBUG [Thread-1191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:22,645 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:42:22,645 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:42:22,654 INFO [Thread-1187 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1187. 2024-12-09T01:42:22,654 DEBUG [Thread-1187 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:42:22,654 DEBUG [Thread-1187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:22,655 DEBUG [Thread-1187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:22,655 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:42:22,750 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=1203 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/c4d2423e06944945bd93c4424d137fc1 2024-12-09T01:42:22,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/1b83f544555045aba75289310f052f6e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/1b83f544555045aba75289310f052f6e 2024-12-09T01:42:22,757 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/1b83f544555045aba75289310f052f6e, entries=150, sequenceid=1203, filesize=12.0 K 2024-12-09T01:42:22,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/541dfd7f37574d11b03e25dccc6a7854 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/541dfd7f37574d11b03e25dccc6a7854 2024-12-09T01:42:22,761 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/541dfd7f37574d11b03e25dccc6a7854, entries=150, sequenceid=1203, filesize=12.0 K 2024-12-09T01:42:22,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/c4d2423e06944945bd93c4424d137fc1 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/c4d2423e06944945bd93c4424d137fc1 2024-12-09T01:42:22,766 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/c4d2423e06944945bd93c4424d137fc1, entries=150, sequenceid=1203, filesize=12.0 K 2024-12-09T01:42:22,768 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=20.13 KB/20610 for bb023e4ee9cc2de03187f3dff0a271ab in 942ms, sequenceid=1203, compaction requested=false 2024-12-09T01:42:22,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for bb023e4ee9cc2de03187f3dff0a271ab: 2024-12-09T01:42:22,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:22,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-12-09T01:42:22,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-12-09T01:42:22,771 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-12-09T01:42:22,771 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3290 sec 2024-12-09T01:42:22,773 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 2.3340 sec 2024-12-09T01:42:23,034 INFO [Thread-1185 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1185. 2024-12-09T01:42:23,034 DEBUG [Thread-1185 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:42:23,034 DEBUG [Thread-1185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:23,035 DEBUG [Thread-1185 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:23,035 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:42:23,045 INFO [Thread-1189 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1189. 2024-12-09T01:42:23,045 DEBUG [Thread-1189 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:42:23,045 DEBUG [Thread-1189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:23,045 DEBUG [Thread-1189 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:23,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:42:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-12-09T01:42:24,604 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:24,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T01:42:24,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 184 2024-12-09T01:42:24,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 232 2024-12-09T01:42:24,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 205 2024-12-09T01:42:24,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 235 2024-12-09T01:42:24,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 243 2024-12-09T01:42:24,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T01:42:24,604 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2297 2024-12-09T01:42:24,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2200 2024-12-09T01:42:24,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2225 2024-12-09T01:42:24,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2275 2024-12-09T01:42:24,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2220 2024-12-09T01:42:24,605 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T01:42:24,605 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T01:42:24,605 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T01:42:24,605 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:24,605 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:24,605 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:42:24,606 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T01:42:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:24,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-09T01:42:24,609 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708544609"}]},"ts":"1733708544609"} 2024-12-09T01:42:24,610 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T01:42:24,610 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T01:42:24,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T01:42:24,612 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bb023e4ee9cc2de03187f3dff0a271ab, UNASSIGN}] 2024-12-09T01:42:24,613 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bb023e4ee9cc2de03187f3dff0a271ab, UNASSIGN 2024-12-09T01:42:24,614 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=bb023e4ee9cc2de03187f3dff0a271ab, regionState=CLOSING, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:24,615 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=bb023e4ee9cc2de03187f3dff0a271ab, UNASSIGN because future has completed 2024-12-09T01:42:24,615 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T01:42:24,615 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943}] 2024-12-09T01:42:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-09T01:42:24,767 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:24,767 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T01:42:24,768 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing bb023e4ee9cc2de03187f3dff0a271ab, disabling compactions & flushes 2024-12-09T01:42:24,768 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:24,768 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:24,768 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. after waiting 0 ms 2024-12-09T01:42:24,768 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:24,768 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(2902): Flushing bb023e4ee9cc2de03187f3dff0a271ab 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-09T01:42:24,768 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=A 2024-12-09T01:42:24,768 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:24,768 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=B 2024-12-09T01:42:24,768 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:24,768 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK bb023e4ee9cc2de03187f3dff0a271ab, store=C 2024-12-09T01:42:24,768 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:24,771 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/407a1d17250d4c7c802ef84a5b74d861 is 50, key is test_row_0/A:col10/1733708542653/Put/seqid=0 2024-12-09T01:42:24,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742191_1367 (size=12301) 2024-12-09T01:42:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-09T01:42:25,176 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1213 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/407a1d17250d4c7c802ef84a5b74d861 2024-12-09T01:42:25,182 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/2e7193e4005e41e29ef4f8126c5c55d3 is 50, key is test_row_0/B:col10/1733708542653/Put/seqid=0 2024-12-09T01:42:25,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742192_1368 (size=12301) 2024-12-09T01:42:25,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-09T01:42:25,585 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1213 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/2e7193e4005e41e29ef4f8126c5c55d3 2024-12-09T01:42:25,591 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/ba38534c34484d4bb7c660714dc1c3ef is 50, key is test_row_0/C:col10/1733708542653/Put/seqid=0 2024-12-09T01:42:25,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742193_1369 (size=12301) 2024-12-09T01:42:25,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-09T01:42:25,995 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1213 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/ba38534c34484d4bb7c660714dc1c3ef 2024-12-09T01:42:25,999 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/A/407a1d17250d4c7c802ef84a5b74d861 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/407a1d17250d4c7c802ef84a5b74d861 2024-12-09T01:42:26,002 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/407a1d17250d4c7c802ef84a5b74d861, entries=150, sequenceid=1213, filesize=12.0 K 2024-12-09T01:42:26,003 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/B/2e7193e4005e41e29ef4f8126c5c55d3 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/2e7193e4005e41e29ef4f8126c5c55d3 2024-12-09T01:42:26,005 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/2e7193e4005e41e29ef4f8126c5c55d3, entries=150, sequenceid=1213, filesize=12.0 K 2024-12-09T01:42:26,006 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/.tmp/C/ba38534c34484d4bb7c660714dc1c3ef as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/ba38534c34484d4bb7c660714dc1c3ef 2024-12-09T01:42:26,009 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/ba38534c34484d4bb7c660714dc1c3ef, entries=150, sequenceid=1213, filesize=12.0 K 2024-12-09T01:42:26,010 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for bb023e4ee9cc2de03187f3dff0a271ab in 1242ms, sequenceid=1213, compaction requested=true 2024-12-09T01:42:26,010 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/8fc706ce5bee4ab1b47e417172cf4d6e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/4bb33189a49940039d0ddb5d39ea3f9f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/3ffff014b671426b8902cd650015b0ad, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/361bed845bcd4a669f5dc5134ee4b3aa, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/a83ff8724a2946268a99205e4a94cbe6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/babd66918b8e42ce97f417d472216c20, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/afd0a777705d424da66135e486c6b79a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/b1982169ca5e417b9334456d15e41b3b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/9fb5e03323964c7c847627ac55e099c1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/bb38bdc20d4c4eb78a09175a6ba33f27, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/a78553a5c9cd4f6688fd4ac37a6064e0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/a6153bfc99ec4eaf8c7b51d500b21ec7, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/18c7bce278e6401286a0725c251d9b72, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/10464651de07473680c03e7c5a38a908, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/6dd7527c155b45d7bd4d2a81e75111a0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/cd4f17e9b834494ca52ead3bc5b68a94, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/06512df454924665943345567286c080, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/253f010e243046cfb9616b7a71ea186c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/865c842be51249a189b7053d91a8dbd8, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/fef93d4987324883bbf9dd569b9bd9ec, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/50f3f59d7c554f1c86283edae7767759, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/6b56e89252eb4606934780dc42dc124e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/8eeb4a3003314c19a2074a61c67fd26f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/0fd0f2d572304324a33573930a51d5b3, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/1e8b04b735074a229ef2092333963108, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/d0038cf0cf0945f0bcc077a757e345f8, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/6495f3b4fc874fcaa4ec441a66c47c8d, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/75844ec575cc4d4c83d8290e646a790a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/8889fe7b6a6e40a9a6200ee160a07de4, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/2a71b0c9e0444db48814117c4b0f8531, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/1ff26ac0c9424be6ac4605aab0bb777f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/4897339f8d7544ce89f618595c06e3ef, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/6e7dd021493c4c41845c1ecd138310af, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/b27424cebe744f49bf49bd6b52e81dea, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/90cd24aa68334738b30a8a5745f37456] to archive 2024-12-09T01:42:26,011 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T01:42:26,013 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/8fc706ce5bee4ab1b47e417172cf4d6e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/8fc706ce5bee4ab1b47e417172cf4d6e 2024-12-09T01:42:26,014 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/4bb33189a49940039d0ddb5d39ea3f9f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/4bb33189a49940039d0ddb5d39ea3f9f 2024-12-09T01:42:26,015 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/3ffff014b671426b8902cd650015b0ad to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/3ffff014b671426b8902cd650015b0ad 2024-12-09T01:42:26,016 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/361bed845bcd4a669f5dc5134ee4b3aa to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/361bed845bcd4a669f5dc5134ee4b3aa 2024-12-09T01:42:26,017 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/a83ff8724a2946268a99205e4a94cbe6 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/a83ff8724a2946268a99205e4a94cbe6 2024-12-09T01:42:26,018 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/babd66918b8e42ce97f417d472216c20 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/babd66918b8e42ce97f417d472216c20 2024-12-09T01:42:26,019 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/afd0a777705d424da66135e486c6b79a to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/afd0a777705d424da66135e486c6b79a 2024-12-09T01:42:26,020 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/b1982169ca5e417b9334456d15e41b3b to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/b1982169ca5e417b9334456d15e41b3b 2024-12-09T01:42:26,021 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/9fb5e03323964c7c847627ac55e099c1 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/9fb5e03323964c7c847627ac55e099c1 2024-12-09T01:42:26,022 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/bb38bdc20d4c4eb78a09175a6ba33f27 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/bb38bdc20d4c4eb78a09175a6ba33f27 2024-12-09T01:42:26,023 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/a78553a5c9cd4f6688fd4ac37a6064e0 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/a78553a5c9cd4f6688fd4ac37a6064e0 2024-12-09T01:42:26,024 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/a6153bfc99ec4eaf8c7b51d500b21ec7 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/a6153bfc99ec4eaf8c7b51d500b21ec7 2024-12-09T01:42:26,025 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/18c7bce278e6401286a0725c251d9b72 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/18c7bce278e6401286a0725c251d9b72 2024-12-09T01:42:26,026 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/10464651de07473680c03e7c5a38a908 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/10464651de07473680c03e7c5a38a908 2024-12-09T01:42:26,027 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/6dd7527c155b45d7bd4d2a81e75111a0 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/6dd7527c155b45d7bd4d2a81e75111a0 2024-12-09T01:42:26,028 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/cd4f17e9b834494ca52ead3bc5b68a94 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/cd4f17e9b834494ca52ead3bc5b68a94 2024-12-09T01:42:26,028 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/06512df454924665943345567286c080 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/06512df454924665943345567286c080 2024-12-09T01:42:26,029 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/253f010e243046cfb9616b7a71ea186c to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/253f010e243046cfb9616b7a71ea186c 2024-12-09T01:42:26,030 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/865c842be51249a189b7053d91a8dbd8 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/865c842be51249a189b7053d91a8dbd8 2024-12-09T01:42:26,031 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/fef93d4987324883bbf9dd569b9bd9ec to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/fef93d4987324883bbf9dd569b9bd9ec 2024-12-09T01:42:26,032 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/50f3f59d7c554f1c86283edae7767759 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/50f3f59d7c554f1c86283edae7767759 2024-12-09T01:42:26,033 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/6b56e89252eb4606934780dc42dc124e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/6b56e89252eb4606934780dc42dc124e 2024-12-09T01:42:26,034 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/8eeb4a3003314c19a2074a61c67fd26f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/8eeb4a3003314c19a2074a61c67fd26f 2024-12-09T01:42:26,035 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/0fd0f2d572304324a33573930a51d5b3 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/0fd0f2d572304324a33573930a51d5b3 2024-12-09T01:42:26,036 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/1e8b04b735074a229ef2092333963108 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/1e8b04b735074a229ef2092333963108 2024-12-09T01:42:26,037 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/d0038cf0cf0945f0bcc077a757e345f8 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/d0038cf0cf0945f0bcc077a757e345f8 2024-12-09T01:42:26,037 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/6495f3b4fc874fcaa4ec441a66c47c8d to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/6495f3b4fc874fcaa4ec441a66c47c8d 2024-12-09T01:42:26,038 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/75844ec575cc4d4c83d8290e646a790a to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/75844ec575cc4d4c83d8290e646a790a 2024-12-09T01:42:26,039 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/8889fe7b6a6e40a9a6200ee160a07de4 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/8889fe7b6a6e40a9a6200ee160a07de4 2024-12-09T01:42:26,040 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/2a71b0c9e0444db48814117c4b0f8531 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/2a71b0c9e0444db48814117c4b0f8531 2024-12-09T01:42:26,041 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/1ff26ac0c9424be6ac4605aab0bb777f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/1ff26ac0c9424be6ac4605aab0bb777f 2024-12-09T01:42:26,041 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/4897339f8d7544ce89f618595c06e3ef to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/4897339f8d7544ce89f618595c06e3ef 2024-12-09T01:42:26,042 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/6e7dd021493c4c41845c1ecd138310af to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/6e7dd021493c4c41845c1ecd138310af 2024-12-09T01:42:26,043 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/b27424cebe744f49bf49bd6b52e81dea to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/b27424cebe744f49bf49bd6b52e81dea 2024-12-09T01:42:26,046 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/90cd24aa68334738b30a8a5745f37456 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/90cd24aa68334738b30a8a5745f37456 2024-12-09T01:42:26,047 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/d66ffb37fe9e49ee9716bb71a1340146, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/babf64e72a7d491e96566552a55de186, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/eb59cf4ef2254848ae82e653ad37c53c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/e38dc64033e447e8a2f8ad955697f47e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/df010469cd09485b9d3be0ad8363b944, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/fbfd255687f846b582208f65bd75cd43, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/eeb673e1bfe44a71839d8a0fd2c7c299, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/046557ea4e9e46e5a84856c009d151ea, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/0aeefa7f4f314efe90733923f600e5de, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/ee79d1767de840f6a70c1919736cd6df, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/ae1ecf9497e3403fac6d45ad43b9ef22, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/f93748e8455b4b7f83d0924faa89a499, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/744493e095dd47cfb6b554a36c7ac0e4, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/d14918fad71f490c9e65d1f778953cac, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/0797cf4f1d504a93b7b2f4cc05bfcec9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/a8dd8d20fd7b42fb9e0cad2dda1318ad, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/3b73c4c2befb4b2b893c54bd27e6e3f2, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/9843148eda3847b8acfdb48443e68502, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/72d8980527004649b33758faefd2bbb4, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/55e23cafd2684477b833475aceefd25a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/30cb3f8bfce7447ab604ec3639d1cb29, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/576e66b71a92455886288c0e9099e443, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/8de5da2b8f1847caa88a400062a39305, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/0b94c6663cdb4deabfaffb5118e09ddc, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/3701244c646142f299c9425850a911d0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/04946f33823c48fdb842af21dc864a15, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/1c04f105f08b4c66b4cb1d82f9f4bef9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/14d3ccc7aaf24a47a4461985ebf383cc, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/8fd12999c354424481ac707d8e9c6748, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/3264ab6e544d46c8987123a3a1797afd, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/7fee44383491414ab4d7398a44c6edee, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/074759a1fb45473a9ee76493471ab6f8, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/71da7995948f4f83aace2f1169847167, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/7a3be0aea9984e478ef4f95e9baa6f91, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/4a55f5b0a0cf407082070d281e76c00c] to archive 2024-12-09T01:42:26,048 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T01:42:26,049 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/d66ffb37fe9e49ee9716bb71a1340146 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/d66ffb37fe9e49ee9716bb71a1340146 2024-12-09T01:42:26,050 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/babf64e72a7d491e96566552a55de186 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/babf64e72a7d491e96566552a55de186 2024-12-09T01:42:26,051 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/eb59cf4ef2254848ae82e653ad37c53c to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/eb59cf4ef2254848ae82e653ad37c53c 2024-12-09T01:42:26,051 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/e38dc64033e447e8a2f8ad955697f47e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/e38dc64033e447e8a2f8ad955697f47e 2024-12-09T01:42:26,052 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/df010469cd09485b9d3be0ad8363b944 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/df010469cd09485b9d3be0ad8363b944 2024-12-09T01:42:26,053 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/fbfd255687f846b582208f65bd75cd43 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/fbfd255687f846b582208f65bd75cd43 2024-12-09T01:42:26,054 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/eeb673e1bfe44a71839d8a0fd2c7c299 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/eeb673e1bfe44a71839d8a0fd2c7c299 2024-12-09T01:42:26,054 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/046557ea4e9e46e5a84856c009d151ea to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/046557ea4e9e46e5a84856c009d151ea 2024-12-09T01:42:26,055 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/0aeefa7f4f314efe90733923f600e5de to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/0aeefa7f4f314efe90733923f600e5de 2024-12-09T01:42:26,056 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/ee79d1767de840f6a70c1919736cd6df to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/ee79d1767de840f6a70c1919736cd6df 2024-12-09T01:42:26,057 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/ae1ecf9497e3403fac6d45ad43b9ef22 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/ae1ecf9497e3403fac6d45ad43b9ef22 2024-12-09T01:42:26,058 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/f93748e8455b4b7f83d0924faa89a499 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/f93748e8455b4b7f83d0924faa89a499 2024-12-09T01:42:26,058 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/744493e095dd47cfb6b554a36c7ac0e4 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/744493e095dd47cfb6b554a36c7ac0e4 2024-12-09T01:42:26,059 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/d14918fad71f490c9e65d1f778953cac to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/d14918fad71f490c9e65d1f778953cac 2024-12-09T01:42:26,060 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/0797cf4f1d504a93b7b2f4cc05bfcec9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/0797cf4f1d504a93b7b2f4cc05bfcec9 2024-12-09T01:42:26,061 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/a8dd8d20fd7b42fb9e0cad2dda1318ad to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/a8dd8d20fd7b42fb9e0cad2dda1318ad 2024-12-09T01:42:26,061 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/3b73c4c2befb4b2b893c54bd27e6e3f2 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/3b73c4c2befb4b2b893c54bd27e6e3f2 2024-12-09T01:42:26,062 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/9843148eda3847b8acfdb48443e68502 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/9843148eda3847b8acfdb48443e68502 2024-12-09T01:42:26,063 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/72d8980527004649b33758faefd2bbb4 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/72d8980527004649b33758faefd2bbb4 2024-12-09T01:42:26,064 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/55e23cafd2684477b833475aceefd25a to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/55e23cafd2684477b833475aceefd25a 2024-12-09T01:42:26,064 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/30cb3f8bfce7447ab604ec3639d1cb29 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/30cb3f8bfce7447ab604ec3639d1cb29 2024-12-09T01:42:26,065 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/576e66b71a92455886288c0e9099e443 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/576e66b71a92455886288c0e9099e443 2024-12-09T01:42:26,066 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/8de5da2b8f1847caa88a400062a39305 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/8de5da2b8f1847caa88a400062a39305 2024-12-09T01:42:26,068 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/0b94c6663cdb4deabfaffb5118e09ddc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/0b94c6663cdb4deabfaffb5118e09ddc 2024-12-09T01:42:26,069 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/3701244c646142f299c9425850a911d0 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/3701244c646142f299c9425850a911d0 2024-12-09T01:42:26,070 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/04946f33823c48fdb842af21dc864a15 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/04946f33823c48fdb842af21dc864a15 2024-12-09T01:42:26,071 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/1c04f105f08b4c66b4cb1d82f9f4bef9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/1c04f105f08b4c66b4cb1d82f9f4bef9 2024-12-09T01:42:26,072 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/14d3ccc7aaf24a47a4461985ebf383cc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/14d3ccc7aaf24a47a4461985ebf383cc 2024-12-09T01:42:26,074 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/8fd12999c354424481ac707d8e9c6748 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/8fd12999c354424481ac707d8e9c6748 2024-12-09T01:42:26,074 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/3264ab6e544d46c8987123a3a1797afd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/3264ab6e544d46c8987123a3a1797afd 2024-12-09T01:42:26,075 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/7fee44383491414ab4d7398a44c6edee to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/7fee44383491414ab4d7398a44c6edee 2024-12-09T01:42:26,076 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/074759a1fb45473a9ee76493471ab6f8 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/074759a1fb45473a9ee76493471ab6f8 2024-12-09T01:42:26,078 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/71da7995948f4f83aace2f1169847167 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/71da7995948f4f83aace2f1169847167 2024-12-09T01:42:26,078 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/7a3be0aea9984e478ef4f95e9baa6f91 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/7a3be0aea9984e478ef4f95e9baa6f91 2024-12-09T01:42:26,079 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/4a55f5b0a0cf407082070d281e76c00c to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/4a55f5b0a0cf407082070d281e76c00c 2024-12-09T01:42:26,081 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/1322642cca5c4fb990edbafe93b8c265, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/53fec10b353247b099377fa976ccc193, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/21550ed2404d4e76949853c707df51cc, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/cd3bdf81fb14484b903856be0c7cfb14, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/1d4878776e784b94931c6febeca6d510, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/67afcbf187cc46f483590567fb671899, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/69319c29107645b5ab58dc50529dbcc1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/c09c0e086ff14789a320ec66acd4be27, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/5e6fd3c6a8dc4b4abaf5f6982e9f6db2, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/0b5250342e9842d682d5808329202c7f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/6a8425f57c3c4d43a8091e0ab7bf5f31, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/c4008d26de184e05a1b9959de7b25981, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/165509d453394ef5bac7e1e18e275cc4, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/00a48995faca4aa3b360003e536e2a14, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/78910a3e966347daafe8e2d2856e2fa0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/e36807d918144bd58732cdb83d44e3aa, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/47a846406c4f4d07b7e4ce7e97a6333f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/9804d761c58e48ffa74f652ed5abfa85, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/4e7adf136c88469ba3c188379da1b2d2, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/9234e3d7e09a4001a8a01794efe1798c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/c22365f5d6b046d68cbe8a160cd1a471, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/9949a1e9228b40ecb2d66240988b57a2, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/0d5c177769b24a8eba972f9bac780846, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/21b90410bfa94fbc9db44ef4d4ecf53f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/4cf99646c207415b96c90525d7b7e50f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/effb8ecae8ac4dd5b818e99f988bb114, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/cf887e0ccabd4276b9f551daefb32a9a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/5dd61c215d6c4be0aac16bd6abe706d6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/a4a88cc38d23465fa7a49995c9de9339, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/8c4055e0d5624b019d9c3d3446a05dfd, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/71bcb1db2d7844759320cb9a6ce87f8e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/b556ea2c768c449a8d9c039d01b39fb2, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/a6139ae70d3845a089ae003e6e1f5733, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/ca4dc3dc68a542e7bae766c406a076ae] to archive 2024-12-09T01:42:26,082 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T01:42:26,083 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/1322642cca5c4fb990edbafe93b8c265 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/1322642cca5c4fb990edbafe93b8c265 2024-12-09T01:42:26,084 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/53fec10b353247b099377fa976ccc193 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/53fec10b353247b099377fa976ccc193 2024-12-09T01:42:26,085 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/21550ed2404d4e76949853c707df51cc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/21550ed2404d4e76949853c707df51cc 2024-12-09T01:42:26,086 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/cd3bdf81fb14484b903856be0c7cfb14 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/cd3bdf81fb14484b903856be0c7cfb14 2024-12-09T01:42:26,087 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/1d4878776e784b94931c6febeca6d510 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/1d4878776e784b94931c6febeca6d510 2024-12-09T01:42:26,088 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/67afcbf187cc46f483590567fb671899 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/67afcbf187cc46f483590567fb671899 2024-12-09T01:42:26,089 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/69319c29107645b5ab58dc50529dbcc1 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/69319c29107645b5ab58dc50529dbcc1 2024-12-09T01:42:26,090 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/c09c0e086ff14789a320ec66acd4be27 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/c09c0e086ff14789a320ec66acd4be27 2024-12-09T01:42:26,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/5e6fd3c6a8dc4b4abaf5f6982e9f6db2 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/5e6fd3c6a8dc4b4abaf5f6982e9f6db2 2024-12-09T01:42:26,091 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/0b5250342e9842d682d5808329202c7f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/0b5250342e9842d682d5808329202c7f 2024-12-09T01:42:26,092 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/6a8425f57c3c4d43a8091e0ab7bf5f31 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/6a8425f57c3c4d43a8091e0ab7bf5f31 2024-12-09T01:42:26,093 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/c4008d26de184e05a1b9959de7b25981 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/c4008d26de184e05a1b9959de7b25981 2024-12-09T01:42:26,094 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/165509d453394ef5bac7e1e18e275cc4 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/165509d453394ef5bac7e1e18e275cc4 2024-12-09T01:42:26,094 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/00a48995faca4aa3b360003e536e2a14 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/00a48995faca4aa3b360003e536e2a14 2024-12-09T01:42:26,095 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/78910a3e966347daafe8e2d2856e2fa0 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/78910a3e966347daafe8e2d2856e2fa0 2024-12-09T01:42:26,096 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/e36807d918144bd58732cdb83d44e3aa to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/e36807d918144bd58732cdb83d44e3aa 2024-12-09T01:42:26,097 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/47a846406c4f4d07b7e4ce7e97a6333f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/47a846406c4f4d07b7e4ce7e97a6333f 2024-12-09T01:42:26,098 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/9804d761c58e48ffa74f652ed5abfa85 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/9804d761c58e48ffa74f652ed5abfa85 2024-12-09T01:42:26,099 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/4e7adf136c88469ba3c188379da1b2d2 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/4e7adf136c88469ba3c188379da1b2d2 2024-12-09T01:42:26,099 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/9234e3d7e09a4001a8a01794efe1798c to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/9234e3d7e09a4001a8a01794efe1798c 2024-12-09T01:42:26,100 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/c22365f5d6b046d68cbe8a160cd1a471 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/c22365f5d6b046d68cbe8a160cd1a471 2024-12-09T01:42:26,101 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/9949a1e9228b40ecb2d66240988b57a2 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/9949a1e9228b40ecb2d66240988b57a2 2024-12-09T01:42:26,102 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/0d5c177769b24a8eba972f9bac780846 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/0d5c177769b24a8eba972f9bac780846 2024-12-09T01:42:26,103 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/21b90410bfa94fbc9db44ef4d4ecf53f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/21b90410bfa94fbc9db44ef4d4ecf53f 2024-12-09T01:42:26,103 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/4cf99646c207415b96c90525d7b7e50f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/4cf99646c207415b96c90525d7b7e50f 2024-12-09T01:42:26,104 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/effb8ecae8ac4dd5b818e99f988bb114 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/effb8ecae8ac4dd5b818e99f988bb114 2024-12-09T01:42:26,105 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/cf887e0ccabd4276b9f551daefb32a9a to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/cf887e0ccabd4276b9f551daefb32a9a 2024-12-09T01:42:26,106 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/5dd61c215d6c4be0aac16bd6abe706d6 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/5dd61c215d6c4be0aac16bd6abe706d6 2024-12-09T01:42:26,107 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/a4a88cc38d23465fa7a49995c9de9339 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/a4a88cc38d23465fa7a49995c9de9339 2024-12-09T01:42:26,107 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/8c4055e0d5624b019d9c3d3446a05dfd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/8c4055e0d5624b019d9c3d3446a05dfd 2024-12-09T01:42:26,108 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/71bcb1db2d7844759320cb9a6ce87f8e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/71bcb1db2d7844759320cb9a6ce87f8e 2024-12-09T01:42:26,109 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/b556ea2c768c449a8d9c039d01b39fb2 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/b556ea2c768c449a8d9c039d01b39fb2 2024-12-09T01:42:26,110 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/a6139ae70d3845a089ae003e6e1f5733 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/a6139ae70d3845a089ae003e6e1f5733 2024-12-09T01:42:26,111 DEBUG [StoreCloser-TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/ca4dc3dc68a542e7bae766c406a076ae to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/ca4dc3dc68a542e7bae766c406a076ae 2024-12-09T01:42:26,115 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/recovered.edits/1216.seqid, newMaxSeqId=1216, maxSeqId=1 2024-12-09T01:42:26,115 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. 2024-12-09T01:42:26,116 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for bb023e4ee9cc2de03187f3dff0a271ab: Waiting for close lock at 1733708544767Running coprocessor pre-close hooks at 1733708544767Disabling compacts and flushes for region at 1733708544768 (+1 ms)Disabling writes for close at 1733708544768Obtaining lock to block concurrent updates at 1733708544768Preparing flush snapshotting stores in bb023e4ee9cc2de03187f3dff0a271ab at 1733708544768Finished memstore snapshotting TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1733708544768Flushing stores of TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab. at 1733708544769 (+1 ms)Flushing bb023e4ee9cc2de03187f3dff0a271ab/A: creating writer at 1733708544769Flushing bb023e4ee9cc2de03187f3dff0a271ab/A: appending metadata at 1733708544771 (+2 ms)Flushing bb023e4ee9cc2de03187f3dff0a271ab/A: closing flushed file at 1733708544771Flushing bb023e4ee9cc2de03187f3dff0a271ab/B: creating writer at 1733708545179 (+408 ms)Flushing bb023e4ee9cc2de03187f3dff0a271ab/B: appending metadata at 1733708545181 (+2 ms)Flushing bb023e4ee9cc2de03187f3dff0a271ab/B: closing flushed file at 1733708545181Flushing bb023e4ee9cc2de03187f3dff0a271ab/C: creating writer at 1733708545589 (+408 ms)Flushing bb023e4ee9cc2de03187f3dff0a271ab/C: appending metadata at 1733708545591 (+2 ms)Flushing bb023e4ee9cc2de03187f3dff0a271ab/C: closing flushed file at 1733708545591Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5046f5c: reopening flushed file at 1733708545998 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54ea02a0: reopening flushed file at 1733708546002 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67e7403e: reopening flushed file at 1733708546006 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for bb023e4ee9cc2de03187f3dff0a271ab in 1242ms, sequenceid=1213, compaction requested=true at 1733708546010 (+4 ms)Writing region close event to WAL at 1733708546112 (+102 ms)Running coprocessor post-close hooks at 1733708546115 (+3 ms)Closed at 1733708546115 2024-12-09T01:42:26,117 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:26,118 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=bb023e4ee9cc2de03187f3dff0a271ab, regionState=CLOSED 2024-12-09T01:42:26,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 because future has completed 2024-12-09T01:42:26,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-12-09T01:42:26,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure bb023e4ee9cc2de03187f3dff0a271ab, server=ef6f18c58dc9,38627,1733708453943 in 1.5050 sec 2024-12-09T01:42:26,123 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-09T01:42:26,123 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=bb023e4ee9cc2de03187f3dff0a271ab, UNASSIGN in 1.5100 sec 2024-12-09T01:42:26,125 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-12-09T01:42:26,125 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5130 sec 2024-12-09T01:42:26,126 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708546126"}]},"ts":"1733708546126"} 2024-12-09T01:42:26,127 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T01:42:26,127 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T01:42:26,129 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5220 sec 2024-12-09T01:42:26,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-12-09T01:42:26,753 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:26,754 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T01:42:26,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:26,756 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:26,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-09T01:42:26,757 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:26,759 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:26,762 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A, FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B, FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C, FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/recovered.edits] 2024-12-09T01:42:26,764 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/1b83f544555045aba75289310f052f6e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/1b83f544555045aba75289310f052f6e 2024-12-09T01:42:26,765 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/34a9a853bf1c428180e2c244ada073e0 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/34a9a853bf1c428180e2c244ada073e0 2024-12-09T01:42:26,766 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/407a1d17250d4c7c802ef84a5b74d861 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/A/407a1d17250d4c7c802ef84a5b74d861 2024-12-09T01:42:26,768 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/2e7193e4005e41e29ef4f8126c5c55d3 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/2e7193e4005e41e29ef4f8126c5c55d3 2024-12-09T01:42:26,769 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/541dfd7f37574d11b03e25dccc6a7854 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/541dfd7f37574d11b03e25dccc6a7854 2024-12-09T01:42:26,770 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/6169009de0c1495eaf131b995b43db67 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/B/6169009de0c1495eaf131b995b43db67 2024-12-09T01:42:26,772 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/45fd21d487594ef5a83989f363447fe8 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/45fd21d487594ef5a83989f363447fe8 2024-12-09T01:42:26,773 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/ba38534c34484d4bb7c660714dc1c3ef to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/ba38534c34484d4bb7c660714dc1c3ef 2024-12-09T01:42:26,774 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/c4d2423e06944945bd93c4424d137fc1 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/C/c4d2423e06944945bd93c4424d137fc1 2024-12-09T01:42:26,777 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/recovered.edits/1216.seqid to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab/recovered.edits/1216.seqid 2024-12-09T01:42:26,777 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/bb023e4ee9cc2de03187f3dff0a271ab 2024-12-09T01:42:26,777 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T01:42:26,779 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:26,782 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T01:42:26,784 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T01:42:26,785 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:26,785 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T01:42:26,785 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733708546785"}]},"ts":"9223372036854775807"} 2024-12-09T01:42:26,787 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T01:42:26,787 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => bb023e4ee9cc2de03187f3dff0a271ab, NAME => 'TestAcidGuarantees,,1733708520393.bb023e4ee9cc2de03187f3dff0a271ab.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T01:42:26,787 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T01:42:26,787 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733708546787"}]},"ts":"9223372036854775807"} 2024-12-09T01:42:26,789 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T01:42:26,789 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:26,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 35 msec 2024-12-09T01:42:26,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-12-09T01:42:26,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T01:42:26,864 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:26,874 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1484270681_22 at /127.0.0.1:32918 [Waiting for operation #284] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1866703801_22 at /127.0.0.1:44646 [Waiting for operation #421] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=619 (was 575) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11835 (was 12129) 2024-12-09T01:42:26,884 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=619, ProcessCount=11, AvailableMemoryMB=11835 2024-12-09T01:42:26,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T01:42:26,885 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:42:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:26,887 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T01:42:26,887 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:26,887 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-12-09T01:42:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T01:42:26,888 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T01:42:26,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742194_1370 (size=1061) 2024-12-09T01:42:26,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T01:42:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T01:42:27,296 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb 2024-12-09T01:42:27,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742195_1371 (size=53) 2024-12-09T01:42:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T01:42:27,701 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:42:27,701 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4b9a7812e88070d8f8c92ba7c915f7b9, disabling compactions & flushes 2024-12-09T01:42:27,701 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:27,701 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:27,701 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. after waiting 0 ms 2024-12-09T01:42:27,701 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:27,701 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:27,701 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4b9a7812e88070d8f8c92ba7c915f7b9: Waiting for close lock at 1733708547701Disabling compacts and flushes for region at 1733708547701Disabling writes for close at 1733708547701Writing region close event to WAL at 1733708547701Closed at 1733708547701 2024-12-09T01:42:27,702 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T01:42:27,703 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733708547702"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733708547702"}]},"ts":"1733708547702"} 2024-12-09T01:42:27,704 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T01:42:27,705 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T01:42:27,705 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708547705"}]},"ts":"1733708547705"} 2024-12-09T01:42:27,706 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T01:42:27,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b9a7812e88070d8f8c92ba7c915f7b9, ASSIGN}] 2024-12-09T01:42:27,708 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b9a7812e88070d8f8c92ba7c915f7b9, ASSIGN 2024-12-09T01:42:27,709 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b9a7812e88070d8f8c92ba7c915f7b9, ASSIGN; state=OFFLINE, location=ef6f18c58dc9,38627,1733708453943; forceNewPlan=false, retain=false 2024-12-09T01:42:27,859 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=4b9a7812e88070d8f8c92ba7c915f7b9, regionState=OPENING, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:27,861 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b9a7812e88070d8f8c92ba7c915f7b9, ASSIGN because future has completed 2024-12-09T01:42:27,861 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943}] 2024-12-09T01:42:28,015 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:28,016 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:42:28,016 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:28,016 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:42:28,016 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:28,016 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:28,017 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:28,018 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:42:28,019 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b9a7812e88070d8f8c92ba7c915f7b9 columnFamilyName A 2024-12-09T01:42:28,019 DEBUG [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:28,019 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] regionserver.HStore(327): Store=4b9a7812e88070d8f8c92ba7c915f7b9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:42:28,019 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:28,020 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:42:28,020 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b9a7812e88070d8f8c92ba7c915f7b9 columnFamilyName B 2024-12-09T01:42:28,020 DEBUG [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:28,021 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] regionserver.HStore(327): Store=4b9a7812e88070d8f8c92ba7c915f7b9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:42:28,021 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:28,022 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:42:28,022 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b9a7812e88070d8f8c92ba7c915f7b9 columnFamilyName C 2024-12-09T01:42:28,022 DEBUG [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:28,022 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] regionserver.HStore(327): Store=4b9a7812e88070d8f8c92ba7c915f7b9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:42:28,022 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:28,022 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:28,023 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:28,023 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T01:42:28,024 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:28,024 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:28,024 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T01:42:28,025 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:28,027 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:42:28,027 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened 4b9a7812e88070d8f8c92ba7c915f7b9; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64967707, jitterRate=-0.031905725598335266}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T01:42:28,027 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:28,028 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for 4b9a7812e88070d8f8c92ba7c915f7b9: Running coprocessor pre-open hook at 1733708548016Writing region info on filesystem at 1733708548016Initializing all the Stores at 1733708548017 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708548017Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708548017Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708548017Cleaning up temporary data from old regions at 1733708548024 (+7 ms)Running coprocessor post-open hooks at 1733708548027 (+3 ms)Region opened successfully at 1733708548028 (+1 ms) 2024-12-09T01:42:28,028 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., pid=93, masterSystemTime=1733708548013 2024-12-09T01:42:28,030 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:28,030 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:28,030 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=4b9a7812e88070d8f8c92ba7c915f7b9, regionState=OPEN, openSeqNum=2, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:28,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 because future has completed 2024-12-09T01:42:28,034 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-12-09T01:42:28,034 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 in 171 msec 2024-12-09T01:42:28,036 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-12-09T01:42:28,036 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b9a7812e88070d8f8c92ba7c915f7b9, ASSIGN in 327 msec 2024-12-09T01:42:28,036 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T01:42:28,037 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708548036"}]},"ts":"1733708548036"} 2024-12-09T01:42:28,038 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T01:42:28,038 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T01:42:28,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1540 sec 2024-12-09T01:42:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-09T01:42:29,033 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:29,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@711d840, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,034 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:29,035 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:29,036 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:29,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:29,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:29,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e90fa01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:29,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:29,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:29,037 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:29,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63891620, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,038 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:29,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:29,040 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:29,040 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T01:42:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is ef6f18c58dc9,37995,1733708453073 2024-12-09T01:42:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5f966458 2024-12-09T01:42:29,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T01:42:29,042 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T01:42:29,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T01:42:29,043 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:42:29,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:29,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742196_1372 (size=1097) 2024-12-09T01:42:29,455 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-09T01:42:29,455 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-09T01:42:29,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T01:42:29,460 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b9a7812e88070d8f8c92ba7c915f7b9, REOPEN/MOVE}] 2024-12-09T01:42:29,460 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b9a7812e88070d8f8c92ba7c915f7b9, REOPEN/MOVE 2024-12-09T01:42:29,461 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=4b9a7812e88070d8f8c92ba7c915f7b9, regionState=CLOSING, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:29,462 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b9a7812e88070d8f8c92ba7c915f7b9, REOPEN/MOVE because future has completed 2024-12-09T01:42:29,463 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T01:42:29,463 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943}] 2024-12-09T01:42:29,615 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:29,615 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T01:42:29,615 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing 4b9a7812e88070d8f8c92ba7c915f7b9, disabling compactions & flushes 2024-12-09T01:42:29,616 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:29,616 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:29,616 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. after waiting 0 ms 2024-12-09T01:42:29,616 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:29,619 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T01:42:29,620 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:29,620 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for 4b9a7812e88070d8f8c92ba7c915f7b9: Waiting for close lock at 1733708549615Running coprocessor pre-close hooks at 1733708549615Disabling compacts and flushes for region at 1733708549615Disabling writes for close at 1733708549616 (+1 ms)Writing region close event to WAL at 1733708549616Running coprocessor post-close hooks at 1733708549619 (+3 ms)Closed at 1733708549620 (+1 ms) 2024-12-09T01:42:29,620 WARN [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: 4b9a7812e88070d8f8c92ba7c915f7b9 to self. 2024-12-09T01:42:29,622 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:29,622 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=4b9a7812e88070d8f8c92ba7c915f7b9, regionState=CLOSED 2024-12-09T01:42:29,624 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 because future has completed 2024-12-09T01:42:29,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-12-09T01:42:29,626 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure 4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 in 161 msec 2024-12-09T01:42:29,626 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b9a7812e88070d8f8c92ba7c915f7b9, REOPEN/MOVE; state=CLOSED, location=ef6f18c58dc9,38627,1733708453943; forceNewPlan=false, retain=true 2024-12-09T01:42:29,777 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=4b9a7812e88070d8f8c92ba7c915f7b9, regionState=OPENING, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:29,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b9a7812e88070d8f8c92ba7c915f7b9, REOPEN/MOVE because future has completed 2024-12-09T01:42:29,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943}] 2024-12-09T01:42:29,933 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:29,933 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:42:29,933 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:29,934 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:42:29,934 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:29,934 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:29,935 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:29,936 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:42:29,936 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b9a7812e88070d8f8c92ba7c915f7b9 columnFamilyName A 2024-12-09T01:42:29,937 DEBUG [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:29,938 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] regionserver.HStore(327): Store=4b9a7812e88070d8f8c92ba7c915f7b9/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:42:29,938 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:29,939 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:42:29,939 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b9a7812e88070d8f8c92ba7c915f7b9 columnFamilyName B 2024-12-09T01:42:29,939 DEBUG [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:29,939 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] regionserver.HStore(327): Store=4b9a7812e88070d8f8c92ba7c915f7b9/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:42:29,939 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:29,940 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:42:29,940 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4b9a7812e88070d8f8c92ba7c915f7b9 columnFamilyName C 2024-12-09T01:42:29,940 DEBUG [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:29,940 INFO [StoreOpener-4b9a7812e88070d8f8c92ba7c915f7b9-1 {}] regionserver.HStore(327): Store=4b9a7812e88070d8f8c92ba7c915f7b9/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:42:29,940 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:29,940 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:29,941 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:29,942 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:29,942 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:29,942 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:29,943 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T01:42:29,944 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:29,945 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened 4b9a7812e88070d8f8c92ba7c915f7b9; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66868367, jitterRate=-0.00358368456363678}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T01:42:29,945 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:29,945 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for 4b9a7812e88070d8f8c92ba7c915f7b9: Running coprocessor pre-open hook at 1733708549934Writing region info on filesystem at 1733708549934Initializing all the Stores at 1733708549934Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708549934Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708549935 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708549935Cleaning up temporary data from old regions at 1733708549942 (+7 ms)Running coprocessor post-open hooks at 1733708549945 (+3 ms)Region opened successfully at 1733708549945 2024-12-09T01:42:29,946 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., pid=98, masterSystemTime=1733708549930 2024-12-09T01:42:29,947 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:29,947 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:29,948 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=4b9a7812e88070d8f8c92ba7c915f7b9, regionState=OPEN, openSeqNum=5, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:29,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 because future has completed 2024-12-09T01:42:29,951 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-12-09T01:42:29,951 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure 4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 in 171 msec 2024-12-09T01:42:29,953 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-09T01:42:29,953 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b9a7812e88070d8f8c92ba7c915f7b9, REOPEN/MOVE in 491 msec 2024-12-09T01:42:29,955 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-12-09T01:42:29,955 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 495 msec 2024-12-09T01:42:29,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 912 msec 2024-12-09T01:42:29,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-12-09T01:42:29,958 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:29,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11853038, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,959 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:29,959 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:29,960 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:29,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:29,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:29,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e6acec4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:29,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:29,960 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:29,961 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:29,962 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e4792aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,962 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c06627f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,962 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:29,962 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:29,963 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:29,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:29,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:29,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cffb33f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:29,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:29,963 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:29,964 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:29,964 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cc65570, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,965 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e8673cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,965 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:29,965 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:29,966 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:29,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:29,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:29,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@217d4ea6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:29,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:29,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:29,967 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36108, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:29,967 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77202757, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18ca1e42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,968 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:29,968 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:29,968 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:29,968 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:29,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:29,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b94381c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:29,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:29,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:29,970 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36120, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:29,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2c8d30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c7eae17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,971 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:29,971 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:29,971 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:29,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:29,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:29,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e90aac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:29,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:29,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:29,972 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:29,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@325455f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,973 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7679e723, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,973 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:29,973 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:29,983 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:29,983 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:29,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4067edfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:29,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:29,984 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:29,985 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:29,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c64ce4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@395008aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,986 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:29,986 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:29,987 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471a1aeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:29,987 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:29,988 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:29,988 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e0fac5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,989 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61351bc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,989 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:29,989 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:29,990 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a5aec27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:29,991 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36202, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:29,991 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@663823a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6444e4e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,992 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:29,992 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:29,992 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:29,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:29,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:29,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@243519f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:29,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:29,993 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:29,994 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:29,994 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7af2dc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b350ef2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,995 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:29,995 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:29,996 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f08b0ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:29,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:29,997 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:29,997 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@318deef3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:29,998 DEBUG [Thread-1685 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:29,998 DEBUG [Thread-1677 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:29,998 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:42:29,999 DEBUG [Thread-1681 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:29,999 DEBUG [Thread-1696 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:29,999 DEBUG [Thread-1679 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:29,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:29,999 DEBUG [Thread-1692 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:29,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:29,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:30,000 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:30,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:30,000 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:30,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:30,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:30,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:30,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:30,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47944, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:30,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:30,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:30,001 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:30,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47968, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:30,001 DEBUG [Thread-1690 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:30,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:30,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-12-09T01:42:30,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:30,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:30,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:30,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T01:42:30,003 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:30,003 DEBUG [Thread-1683 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:30,003 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:42:30,003 DEBUG [Thread-1694 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:30,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:30,004 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:42:30,004 DEBUG [Thread-1688 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:30,004 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:42:30,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:42:30,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:42:30,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:30,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:30,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:42:30,006 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:30,007 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:42:30,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:42:30,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:42:30,008 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:42:30,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:42:30,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:30,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:30,010 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:30,011 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:42:30,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:30,012 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:30,013 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:30,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:42:30,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:30,022 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9a7812e88070d8f8c92ba7c915f7b9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T01:42:30,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=A 2024-12-09T01:42:30,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:30,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=B 2024-12-09T01:42:30,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:30,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=C 2024-12-09T01:42:30,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:30,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209aa820cd4d4d047fa8cf4d47097f302cc_4b9a7812e88070d8f8c92ba7c915f7b9 is 50, key is test_row_0/A:col10/1733708550019/Put/seqid=0 2024-12-09T01:42:30,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742197_1373 (size=14594) 2024-12-09T01:42:30,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:30,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T01:42:30,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708610099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:30,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:30,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:30,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708610100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:30,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:30,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,114 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:30,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708610101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:30,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:30,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:30,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708610107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:30,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,116 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:30,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:30,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708610113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:30,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,126 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:30,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T01:42:30,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:30,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:30,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:30,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:30,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:30,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:30,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:30,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708610224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:30,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:30,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708610224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:30,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:30,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:30,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708610224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:30,230 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:30,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:30,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708610224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,231 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:30,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:30,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708610234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:30,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T01:42:30,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:30,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:30,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:30,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:30,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:30,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:30,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T01:42:30,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:30,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708610433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:30,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708610433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:30,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:30,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708610434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:30,443 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:30,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:30,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708610444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:30,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:30,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:30,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708610453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,456 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:30,456 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:30,460 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209aa820cd4d4d047fa8cf4d47097f302cc_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209aa820cd4d4d047fa8cf4d47097f302cc_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:30,461 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/3eaa7d2380414f249a73ca57ac671d69, store: [table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:30,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/3eaa7d2380414f249a73ca57ac671d69 is 175, key is test_row_0/A:col10/1733708550019/Put/seqid=0 2024-12-09T01:42:30,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742198_1374 (size=39549) 2024-12-09T01:42:30,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T01:42:30,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:30,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:30,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:30,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:30,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:30,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:30,470 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/3eaa7d2380414f249a73ca57ac671d69 2024-12-09T01:42:30,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/2b319aceac1d472891fd0329c33735b6 is 50, key is test_row_0/B:col10/1733708550019/Put/seqid=0 2024-12-09T01:42:30,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742199_1375 (size=12001) 2024-12-09T01:42:30,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T01:42:30,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:30,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:30,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:30,620 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:30,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:30,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:30,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T01:42:30,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:30,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708610754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:30,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708610754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:30,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708610754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:30,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:30,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708610763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:30,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:30,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:30,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708610763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:30,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:30,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:30,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T01:42:30,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:30,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:30,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:30,773 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:30,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:30,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:30,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/2b319aceac1d472891fd0329c33735b6 2024-12-09T01:42:30,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T01:42:30,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:30,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:30,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:30,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:30,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:30,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:30,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/0cbcb35c0e804541b46713e0f5ec47c0 is 50, key is test_row_0/C:col10/1733708550019/Put/seqid=0 2024-12-09T01:42:30,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742200_1376 (size=12001) 2024-12-09T01:42:31,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T01:42:31,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:31,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:31,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:31,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:31,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:31,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T01:42:31,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T01:42:31,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:31,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:31,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:31,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:31,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:31,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:31,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708611273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:31,279 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:31,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:31,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708611274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:31,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:31,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:31,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:31,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:31,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708611274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:31,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:31,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:31,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:31,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:31,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708611274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:31,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:31,282 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:31,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:31,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:31,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708611274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:31,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:31,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:31,283 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:31,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/0cbcb35c0e804541b46713e0f5ec47c0 2024-12-09T01:42:31,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/3eaa7d2380414f249a73ca57ac671d69 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3eaa7d2380414f249a73ca57ac671d69 2024-12-09T01:42:31,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3eaa7d2380414f249a73ca57ac671d69, entries=200, sequenceid=16, filesize=38.6 K 2024-12-09T01:42:31,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/2b319aceac1d472891fd0329c33735b6 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/2b319aceac1d472891fd0329c33735b6 2024-12-09T01:42:31,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/2b319aceac1d472891fd0329c33735b6, entries=150, sequenceid=16, filesize=11.7 K 2024-12-09T01:42:31,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/0cbcb35c0e804541b46713e0f5ec47c0 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/0cbcb35c0e804541b46713e0f5ec47c0 2024-12-09T01:42:31,354 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/0cbcb35c0e804541b46713e0f5ec47c0, entries=150, sequenceid=16, filesize=11.7 K 2024-12-09T01:42:31,355 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4b9a7812e88070d8f8c92ba7c915f7b9 in 1333ms, sequenceid=16, compaction requested=false 2024-12-09T01:42:31,355 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-12-09T01:42:31,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:31,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-12-09T01:42:31,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:31,387 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 4b9a7812e88070d8f8c92ba7c915f7b9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T01:42:31,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=A 2024-12-09T01:42:31,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:31,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=B 2024-12-09T01:42:31,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:31,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=C 2024-12-09T01:42:31,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:31,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209eb0a447a44204e3ea11f1f25a78aa016_4b9a7812e88070d8f8c92ba7c915f7b9 is 50, key is test_row_0/A:col10/1733708550105/Put/seqid=0 2024-12-09T01:42:31,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742201_1377 (size=12154) 2024-12-09T01:42:31,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:31,413 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209eb0a447a44204e3ea11f1f25a78aa016_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209eb0a447a44204e3ea11f1f25a78aa016_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:31,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/18ffadd011774f129bfd8ee6b55d27c9, store: [table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:31,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/18ffadd011774f129bfd8ee6b55d27c9 is 175, key is test_row_0/A:col10/1733708550105/Put/seqid=0 2024-12-09T01:42:31,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742202_1378 (size=30955) 2024-12-09T01:42:31,644 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T01:42:31,821 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/18ffadd011774f129bfd8ee6b55d27c9 2024-12-09T01:42:31,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/a32ab115f9e449968f9e1d2e45fe4034 is 50, key is test_row_0/B:col10/1733708550105/Put/seqid=0 2024-12-09T01:42:31,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742203_1379 (size=12001) 2024-12-09T01:42:32,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T01:42:32,252 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/a32ab115f9e449968f9e1d2e45fe4034 2024-12-09T01:42:32,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/62f333f2217d4df99f83f320a034fda4 is 50, key is test_row_0/C:col10/1733708550105/Put/seqid=0 2024-12-09T01:42:32,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742204_1380 (size=12001) 2024-12-09T01:42:32,306 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/62f333f2217d4df99f83f320a034fda4 2024-12-09T01:42:32,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/18ffadd011774f129bfd8ee6b55d27c9 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/18ffadd011774f129bfd8ee6b55d27c9 2024-12-09T01:42:32,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:32,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:32,316 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/18ffadd011774f129bfd8ee6b55d27c9, entries=150, sequenceid=36, filesize=30.2 K 2024-12-09T01:42:32,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/a32ab115f9e449968f9e1d2e45fe4034 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/a32ab115f9e449968f9e1d2e45fe4034 2024-12-09T01:42:32,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,328 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/a32ab115f9e449968f9e1d2e45fe4034, entries=150, sequenceid=36, filesize=11.7 K 2024-12-09T01:42:32,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/62f333f2217d4df99f83f320a034fda4 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/62f333f2217d4df99f83f320a034fda4 2024-12-09T01:42:32,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,335 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/62f333f2217d4df99f83f320a034fda4, entries=150, sequenceid=36, filesize=11.7 K 2024-12-09T01:42:32,336 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=33.54 KB/34350 for 4b9a7812e88070d8f8c92ba7c915f7b9 in 949ms, sequenceid=36, compaction requested=false 2024-12-09T01:42:32,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:32,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:32,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-09T01:42:32,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-12-09T01:42:32,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:32,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9a7812e88070d8f8c92ba7c915f7b9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T01:42:32,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=A 2024-12-09T01:42:32,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:32,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=B 2024-12-09T01:42:32,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:32,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=C 2024-12-09T01:42:32,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:32,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-12-09T01:42:32,343 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3350 sec 2024-12-09T01:42:32,347 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.3460 sec 2024-12-09T01:42:32,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209adb74fd30cc249458a42a68c386853e7_4b9a7812e88070d8f8c92ba7c915f7b9 is 50, key is test_row_0/A:col10/1733708552306/Put/seqid=0 2024-12-09T01:42:32,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742205_1381 (size=21918) 2024-12-09T01:42:32,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,391 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:32,395 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209adb74fd30cc249458a42a68c386853e7_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209adb74fd30cc249458a42a68c386853e7_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:32,396 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/3cac59c4c1bf4018860b3a86eb5d1ccf, store: [table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:32,397 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/3cac59c4c1bf4018860b3a86eb5d1ccf is 175, key is test_row_0/A:col10/1733708552306/Put/seqid=0 2024-12-09T01:42:32,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742206_1382 (size=65323) 2024-12-09T01:42:32,420 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=47, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/3cac59c4c1bf4018860b3a86eb5d1ccf 2024-12-09T01:42:32,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708552409, max timestamp=1733708552422Num uniques 859; 2024-12-09T01:42:32,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:42:32,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708552409, max timestamp=1733708552422Num uniques 859; 2024-12-09T01:42:32,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:42:32,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708552409, max timestamp=1733708552422Num uniques 859; 2024-12-09T01:42:32,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:42:32,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/e92ad5d8c4d045c889ffaa9e565c6107 is 50, key is test_row_0/B:col10/1733708552306/Put/seqid=0 2024-12-09T01:42:32,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742207_1383 (size=12001) 2024-12-09T01:42:32,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:32,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708612468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:32,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:32,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:32,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708612476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:32,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:32,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708612477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:32,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:32,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708612478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,498 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:32,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:32,503 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:32,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708612484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:32,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:32,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708612593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:32,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:32,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:32,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708612603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:32,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:32,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708612604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:32,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:32,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:32,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708612604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:32,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:32,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:32,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708612613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:32,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:32,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708612814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:32,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:32,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:32,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708612823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:32,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:32,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708612823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:32,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:32,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708612823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:32,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:32,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,838 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:32,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/e92ad5d8c4d045c889ffaa9e565c6107 2024-12-09T01:42:32,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:32,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708612834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:32,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:32,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:32,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/959ec43b49554ca499e6de3b83d99f8e is 50, key is test_row_0/C:col10/1733708552306/Put/seqid=0 2024-12-09T01:42:32,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742208_1384 (size=12001) 2024-12-09T01:42:32,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/959ec43b49554ca499e6de3b83d99f8e 2024-12-09T01:42:32,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/3cac59c4c1bf4018860b3a86eb5d1ccf as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3cac59c4c1bf4018860b3a86eb5d1ccf 2024-12-09T01:42:32,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3cac59c4c1bf4018860b3a86eb5d1ccf, entries=350, sequenceid=47, filesize=63.8 K 2024-12-09T01:42:32,894 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/e92ad5d8c4d045c889ffaa9e565c6107 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/e92ad5d8c4d045c889ffaa9e565c6107 2024-12-09T01:42:32,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/e92ad5d8c4d045c889ffaa9e565c6107, entries=150, sequenceid=47, filesize=11.7 K 2024-12-09T01:42:32,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/959ec43b49554ca499e6de3b83d99f8e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/959ec43b49554ca499e6de3b83d99f8e 2024-12-09T01:42:32,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/959ec43b49554ca499e6de3b83d99f8e, entries=150, sequenceid=47, filesize=11.7 K 2024-12-09T01:42:32,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 4b9a7812e88070d8f8c92ba7c915f7b9 in 569ms, sequenceid=47, compaction requested=true 2024-12-09T01:42:32,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:32,908 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:32,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9a7812e88070d8f8c92ba7c915f7b9:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:42:32,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:32,909 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:32,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9a7812e88070d8f8c92ba7c915f7b9:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:42:32,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:32,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9a7812e88070d8f8c92ba7c915f7b9:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:42:32,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:32,910 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:32,910 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135827 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:32,910 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 4b9a7812e88070d8f8c92ba7c915f7b9/A is initiating minor compaction (all files) 2024-12-09T01:42:32,910 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 4b9a7812e88070d8f8c92ba7c915f7b9/B is initiating minor compaction (all files) 2024-12-09T01:42:32,910 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9a7812e88070d8f8c92ba7c915f7b9/A in TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:32,910 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9a7812e88070d8f8c92ba7c915f7b9/B in TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:32,910 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3eaa7d2380414f249a73ca57ac671d69, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/18ffadd011774f129bfd8ee6b55d27c9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3cac59c4c1bf4018860b3a86eb5d1ccf] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp, totalSize=132.6 K 2024-12-09T01:42:32,910 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/2b319aceac1d472891fd0329c33735b6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/a32ab115f9e449968f9e1d2e45fe4034, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/e92ad5d8c4d045c889ffaa9e565c6107] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp, totalSize=35.2 K 2024-12-09T01:42:32,910 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:32,910 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. files: [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3eaa7d2380414f249a73ca57ac671d69, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/18ffadd011774f129bfd8ee6b55d27c9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3cac59c4c1bf4018860b3a86eb5d1ccf] 2024-12-09T01:42:32,911 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b319aceac1d472891fd0329c33735b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733708550005 2024-12-09T01:42:32,911 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eaa7d2380414f249a73ca57ac671d69, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733708550005 2024-12-09T01:42:32,912 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting a32ab115f9e449968f9e1d2e45fe4034, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733708550100 2024-12-09T01:42:32,912 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18ffadd011774f129bfd8ee6b55d27c9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733708550100 2024-12-09T01:42:32,913 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting e92ad5d8c4d045c889ffaa9e565c6107, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733708552306 2024-12-09T01:42:32,913 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cac59c4c1bf4018860b3a86eb5d1ccf, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733708552300 2024-12-09T01:42:32,933 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:32,934 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9a7812e88070d8f8c92ba7c915f7b9#B#compaction#335 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:32,935 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412093df1c15be77c45ea82995e783c30d99a_4b9a7812e88070d8f8c92ba7c915f7b9 store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:32,935 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/b5b11f7000d5436ea9432668c4100c13 is 50, key is test_row_0/B:col10/1733708552306/Put/seqid=0 2024-12-09T01:42:32,937 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412093df1c15be77c45ea82995e783c30d99a_4b9a7812e88070d8f8c92ba7c915f7b9, store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:32,937 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412093df1c15be77c45ea82995e783c30d99a_4b9a7812e88070d8f8c92ba7c915f7b9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:32,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742209_1385 (size=4469) 2024-12-09T01:42:33,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742210_1386 (size=12104) 2024-12-09T01:42:33,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:33,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9a7812e88070d8f8c92ba7c915f7b9 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T01:42:33,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=A 2024-12-09T01:42:33,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:33,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=B 2024-12-09T01:42:33,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:33,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=C 2024-12-09T01:42:33,146 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:33,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120988bdfff3ec9b4068a0e29a239e791ba1_4b9a7812e88070d8f8c92ba7c915f7b9 is 50, key is test_row_0/A:col10/1733708553143/Put/seqid=0 2024-12-09T01:42:33,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742211_1387 (size=14594) 2024-12-09T01:42:33,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:33,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708613159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:33,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:33,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708613175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:33,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:33,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708613177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:33,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:33,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:33,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708613180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:33,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:33,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708613180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:33,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:33,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:33,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:33,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708613284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:33,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:33,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708613303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:33,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:33,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,317 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:33,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708613303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:33,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:33,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:33,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708613304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:33,317 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:33,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708613304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:33,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:33,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:33,360 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9a7812e88070d8f8c92ba7c915f7b9#A#compaction#336 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:33,361 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/8a07a28fa0f74cada5b7908a54a0a3df is 175, key is test_row_0/A:col10/1733708552306/Put/seqid=0 2024-12-09T01:42:33,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742212_1388 (size=31058) 2024-12-09T01:42:33,406 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/b5b11f7000d5436ea9432668c4100c13 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/b5b11f7000d5436ea9432668c4100c13 2024-12-09T01:42:33,413 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b9a7812e88070d8f8c92ba7c915f7b9/B of 4b9a7812e88070d8f8c92ba7c915f7b9 into b5b11f7000d5436ea9432668c4100c13(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:33,413 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:33,413 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., storeName=4b9a7812e88070d8f8c92ba7c915f7b9/B, priority=13, startTime=1733708552909; duration=0sec 2024-12-09T01:42:33,413 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:33,413 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9a7812e88070d8f8c92ba7c915f7b9:B 2024-12-09T01:42:33,413 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:33,414 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:33,414 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 4b9a7812e88070d8f8c92ba7c915f7b9/C is initiating minor compaction (all files) 2024-12-09T01:42:33,414 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9a7812e88070d8f8c92ba7c915f7b9/C in TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:33,414 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/0cbcb35c0e804541b46713e0f5ec47c0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/62f333f2217d4df99f83f320a034fda4, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/959ec43b49554ca499e6de3b83d99f8e] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp, totalSize=35.2 K 2024-12-09T01:42:33,415 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cbcb35c0e804541b46713e0f5ec47c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1733708550005 2024-12-09T01:42:33,415 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 62f333f2217d4df99f83f320a034fda4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733708550100 2024-12-09T01:42:33,416 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 959ec43b49554ca499e6de3b83d99f8e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733708552306 2024-12-09T01:42:33,426 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9a7812e88070d8f8c92ba7c915f7b9#C#compaction#338 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:33,427 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/9d5ad637ce5c421f92dda57e61e61d27 is 50, key is test_row_0/C:col10/1733708552306/Put/seqid=0 2024-12-09T01:42:33,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742213_1389 (size=12104) 2024-12-09T01:42:33,446 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/9d5ad637ce5c421f92dda57e61e61d27 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/9d5ad637ce5c421f92dda57e61e61d27 2024-12-09T01:42:33,450 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b9a7812e88070d8f8c92ba7c915f7b9/C of 4b9a7812e88070d8f8c92ba7c915f7b9 into 9d5ad637ce5c421f92dda57e61e61d27(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:33,450 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:33,450 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., storeName=4b9a7812e88070d8f8c92ba7c915f7b9/C, priority=13, startTime=1733708552909; duration=0sec 2024-12-09T01:42:33,450 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:33,450 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9a7812e88070d8f8c92ba7c915f7b9:C 2024-12-09T01:42:33,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:33,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708613493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:33,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708613523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:33,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:33,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708613524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:33,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,532 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:33,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:33,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708613524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:33,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:33,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:33,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:33,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708613524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,533 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:33,575 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:33,578 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120988bdfff3ec9b4068a0e29a239e791ba1_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120988bdfff3ec9b4068a0e29a239e791ba1_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:33,579 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/e3645cd605704de8b58529fcfe8b2a8e, store: [table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:33,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/e3645cd605704de8b58529fcfe8b2a8e is 175, key is test_row_0/A:col10/1733708553143/Put/seqid=0 2024-12-09T01:42:33,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742214_1390 (size=39549) 2024-12-09T01:42:33,776 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/8a07a28fa0f74cada5b7908a54a0a3df as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/8a07a28fa0f74cada5b7908a54a0a3df 2024-12-09T01:42:33,780 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b9a7812e88070d8f8c92ba7c915f7b9/A of 4b9a7812e88070d8f8c92ba7c915f7b9 into 8a07a28fa0f74cada5b7908a54a0a3df(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:33,780 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:33,780 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., storeName=4b9a7812e88070d8f8c92ba7c915f7b9/A, priority=13, startTime=1733708552908; duration=0sec 2024-12-09T01:42:33,780 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:33,780 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9a7812e88070d8f8c92ba7c915f7b9:A 2024-12-09T01:42:33,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:33,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708613814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:33,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:33,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708613834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:33,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:33,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708613843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:33,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:33,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,855 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:33,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708613843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:33,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:33,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708613843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:33,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:33,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:33,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:33,984 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/e3645cd605704de8b58529fcfe8b2a8e 2024-12-09T01:42:33,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/5960d5885b2d4974b7d11b2f8657a409 is 50, key is test_row_0/B:col10/1733708553143/Put/seqid=0 2024-12-09T01:42:33,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742215_1391 (size=12001) 2024-12-09T01:42:34,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-09T01:42:34,164 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:34,165 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:42:34,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-12-09T01:42:34,167 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:42:34,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T01:42:34,168 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:42:34,168 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:42:34,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T01:42:34,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T01:42:34,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:34,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:34,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:34,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:34,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:34,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:34,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:34,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708614336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:34,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:34,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:34,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:34,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:34,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708614344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:34,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:34,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:34,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:34,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:34,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708614363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:34,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:34,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:34,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708614363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:34,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:34,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:34,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708614363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:34,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:34,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:34,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:34,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:34,376 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:34,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/5960d5885b2d4974b7d11b2f8657a409 2024-12-09T01:42:34,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/f8812d756ab94cb391bcbc7a9cde01a0 is 50, key is test_row_0/C:col10/1733708553143/Put/seqid=0 2024-12-09T01:42:34,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742216_1392 (size=12001) 2024-12-09T01:42:34,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/f8812d756ab94cb391bcbc7a9cde01a0 2024-12-09T01:42:34,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/e3645cd605704de8b58529fcfe8b2a8e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/e3645cd605704de8b58529fcfe8b2a8e 2024-12-09T01:42:34,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/e3645cd605704de8b58529fcfe8b2a8e, entries=200, sequenceid=83, filesize=38.6 K 2024-12-09T01:42:34,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/5960d5885b2d4974b7d11b2f8657a409 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/5960d5885b2d4974b7d11b2f8657a409 2024-12-09T01:42:34,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/5960d5885b2d4974b7d11b2f8657a409, entries=150, sequenceid=83, filesize=11.7 K 2024-12-09T01:42:34,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/f8812d756ab94cb391bcbc7a9cde01a0 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/f8812d756ab94cb391bcbc7a9cde01a0 2024-12-09T01:42:34,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/f8812d756ab94cb391bcbc7a9cde01a0, entries=150, sequenceid=83, filesize=11.7 K 2024-12-09T01:42:34,451 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for 4b9a7812e88070d8f8c92ba7c915f7b9 in 1306ms, sequenceid=83, compaction requested=false 2024-12-09T01:42:34,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:34,475 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-12-09T01:42:34,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:34,475 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 4b9a7812e88070d8f8c92ba7c915f7b9 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T01:42:34,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=A 2024-12-09T01:42:34,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:34,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=B 2024-12-09T01:42:34,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:34,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=C 2024-12-09T01:42:34,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:34,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T01:42:34,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120984ffbc99d00d49f696bbe9e08364e304_4b9a7812e88070d8f8c92ba7c915f7b9 is 50, key is test_row_0/A:col10/1733708553171/Put/seqid=0 2024-12-09T01:42:34,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742217_1393 (size=12154) 2024-12-09T01:42:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T01:42:34,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:34,902 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120984ffbc99d00d49f696bbe9e08364e304_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120984ffbc99d00d49f696bbe9e08364e304_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:34,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/6e64cc14fbdc4c4fb51905273680a2ee, store: [table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:34,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/6e64cc14fbdc4c4fb51905273680a2ee is 175, key is test_row_0/A:col10/1733708553171/Put/seqid=0 2024-12-09T01:42:34,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742218_1394 (size=30955) 2024-12-09T01:42:34,909 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/6e64cc14fbdc4c4fb51905273680a2ee 2024-12-09T01:42:34,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/b7494de9f7444c8d83457c32f4245380 is 50, key is test_row_0/B:col10/1733708553171/Put/seqid=0 2024-12-09T01:42:34,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742219_1395 (size=12001) 2024-12-09T01:42:35,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T01:42:35,326 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/b7494de9f7444c8d83457c32f4245380 2024-12-09T01:42:35,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/fb9e0a3ae23d49e8b00d74fe275c2a4f is 50, key is test_row_0/C:col10/1733708553171/Put/seqid=0 2024-12-09T01:42:35,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742220_1396 (size=12001) 2024-12-09T01:42:35,348 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/fb9e0a3ae23d49e8b00d74fe275c2a4f 2024-12-09T01:42:35,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/6e64cc14fbdc4c4fb51905273680a2ee as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/6e64cc14fbdc4c4fb51905273680a2ee 2024-12-09T01:42:35,358 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/6e64cc14fbdc4c4fb51905273680a2ee, entries=150, sequenceid=97, filesize=30.2 K 2024-12-09T01:42:35,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/b7494de9f7444c8d83457c32f4245380 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/b7494de9f7444c8d83457c32f4245380 2024-12-09T01:42:35,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:35,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:35,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,363 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/b7494de9f7444c8d83457c32f4245380, entries=150, sequenceid=97, filesize=11.7 K 2024-12-09T01:42:35,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/fb9e0a3ae23d49e8b00d74fe275c2a4f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/fb9e0a3ae23d49e8b00d74fe275c2a4f 2024-12-09T01:42:35,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,368 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/fb9e0a3ae23d49e8b00d74fe275c2a4f, entries=150, sequenceid=97, filesize=11.7 K 2024-12-09T01:42:35,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,370 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=6.71 KB/6870 for 4b9a7812e88070d8f8c92ba7c915f7b9 in 894ms, sequenceid=97, compaction requested=true 2024-12-09T01:42:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:35,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:35,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-12-09T01:42:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-12-09T01:42:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,373 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-12-09T01:42:35,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,373 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2030 sec 2024-12-09T01:42:35,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,375 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.2090 sec 2024-12-09T01:42:35,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:35,494 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9a7812e88070d8f8c92ba7c915f7b9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:42:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=A 2024-12-09T01:42:35,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:35,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=B 2024-12-09T01:42:35,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:35,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=C 2024-12-09T01:42:35,495 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:35,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d58fbccac1d0472bb661c3eec4de567b_4b9a7812e88070d8f8c92ba7c915f7b9 is 50, key is test_row_0/A:col10/1733708555473/Put/seqid=0 2024-12-09T01:42:35,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742221_1397 (size=21918) 2024-12-09T01:42:35,540 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:35,548 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d58fbccac1d0472bb661c3eec4de567b_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d58fbccac1d0472bb661c3eec4de567b_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:35,549 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/622fd52003d340f7ab570f853ab659ba, store: [table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:35,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/622fd52003d340f7ab570f853ab659ba is 175, key is test_row_0/A:col10/1733708555473/Put/seqid=0 2024-12-09T01:42:35,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708555551, max timestamp=1733708555572Num uniques 859; 2024-12-09T01:42:35,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708555551, max timestamp=1733708555572Num uniques 859; 2024-12-09T01:42:35,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:42:35,578 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:42:35,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708555551, max timestamp=1733708555572Num uniques 859; 2024-12-09T01:42:35,580 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:42:35,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742222_1398 (size=65323) 2024-12-09T01:42:35,584 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=107, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/622fd52003d340f7ab570f853ab659ba 2024-12-09T01:42:35,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/3ef536540d8a4c88a93816e0ac58d095 is 50, key is test_row_0/B:col10/1733708555473/Put/seqid=0 2024-12-09T01:42:35,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742223_1399 (size=12001) 2024-12-09T01:42:35,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708615628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:35,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:35,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708615629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:35,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:35,647 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:35,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:35,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708615630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:35,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:35,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708615630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:35,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:35,652 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:35,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708615631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:35,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:35,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708615754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:35,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:35,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708615754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:35,768 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:35,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:35,769 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:35,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708615754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:35,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:35,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708615763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:35,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:35,774 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:35,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708615763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:35,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:35,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708615973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:35,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708615973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:35,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:35,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708615973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,985 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:35,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708615984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:35,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:35,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:35,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708615984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:35,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:36,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/3ef536540d8a4c88a93816e0ac58d095 2024-12-09T01:42:36,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/3da68982540943708c24e175ef37a065 is 50, key is test_row_0/C:col10/1733708555473/Put/seqid=0 2024-12-09T01:42:36,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742224_1400 (size=12001) 2024-12-09T01:42:36,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/3da68982540943708c24e175ef37a065 2024-12-09T01:42:36,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/622fd52003d340f7ab570f853ab659ba as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/622fd52003d340f7ab570f853ab659ba 2024-12-09T01:42:36,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/622fd52003d340f7ab570f853ab659ba, entries=350, sequenceid=107, filesize=63.8 K 2024-12-09T01:42:36,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/3ef536540d8a4c88a93816e0ac58d095 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/3ef536540d8a4c88a93816e0ac58d095 2024-12-09T01:42:36,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/3ef536540d8a4c88a93816e0ac58d095, entries=150, sequenceid=107, filesize=11.7 K 2024-12-09T01:42:36,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/3da68982540943708c24e175ef37a065 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/3da68982540943708c24e175ef37a065 2024-12-09T01:42:36,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/3da68982540943708c24e175ef37a065, entries=150, sequenceid=107, filesize=11.7 K 2024-12-09T01:42:36,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 4b9a7812e88070d8f8c92ba7c915f7b9 in 545ms, sequenceid=107, compaction requested=true 2024-12-09T01:42:36,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:36,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9a7812e88070d8f8c92ba7c915f7b9:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:42:36,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:36,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9a7812e88070d8f8c92ba7c915f7b9:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:42:36,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:36,040 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:42:36,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9a7812e88070d8f8c92ba7c915f7b9:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:42:36,040 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:36,040 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:42:36,041 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 166885 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:42:36,041 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 4b9a7812e88070d8f8c92ba7c915f7b9/A is initiating minor compaction (all files) 2024-12-09T01:42:36,041 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9a7812e88070d8f8c92ba7c915f7b9/A in TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:36,041 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:42:36,041 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 4b9a7812e88070d8f8c92ba7c915f7b9/B is initiating minor compaction (all files) 2024-12-09T01:42:36,041 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/8a07a28fa0f74cada5b7908a54a0a3df, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/e3645cd605704de8b58529fcfe8b2a8e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/6e64cc14fbdc4c4fb51905273680a2ee, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/622fd52003d340f7ab570f853ab659ba] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp, totalSize=163.0 K 2024-12-09T01:42:36,041 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:36,041 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9a7812e88070d8f8c92ba7c915f7b9/B in TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:36,041 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. files: [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/8a07a28fa0f74cada5b7908a54a0a3df, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/e3645cd605704de8b58529fcfe8b2a8e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/6e64cc14fbdc4c4fb51905273680a2ee, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/622fd52003d340f7ab570f853ab659ba] 2024-12-09T01:42:36,041 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/b5b11f7000d5436ea9432668c4100c13, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/5960d5885b2d4974b7d11b2f8657a409, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/b7494de9f7444c8d83457c32f4245380, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/3ef536540d8a4c88a93816e0ac58d095] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp, totalSize=47.0 K 2024-12-09T01:42:36,042 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a07a28fa0f74cada5b7908a54a0a3df, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733708552306 2024-12-09T01:42:36,042 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting b5b11f7000d5436ea9432668c4100c13, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733708552306 2024-12-09T01:42:36,042 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3645cd605704de8b58529fcfe8b2a8e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733708552464 2024-12-09T01:42:36,042 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 5960d5885b2d4974b7d11b2f8657a409, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733708552464 2024-12-09T01:42:36,042 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e64cc14fbdc4c4fb51905273680a2ee, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733708553171 2024-12-09T01:42:36,042 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting b7494de9f7444c8d83457c32f4245380, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733708553171 2024-12-09T01:42:36,042 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 622fd52003d340f7ab570f853ab659ba, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733708555360 2024-12-09T01:42:36,042 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ef536540d8a4c88a93816e0ac58d095, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733708555472 2024-12-09T01:42:36,057 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9a7812e88070d8f8c92ba7c915f7b9#B#compaction#347 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:36,057 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/6c9ceb64995049628f721870cefc7db0 is 50, key is test_row_0/B:col10/1733708555473/Put/seqid=0 2024-12-09T01:42:36,060 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:36,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742225_1401 (size=12241) 2024-12-09T01:42:36,064 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209190869ad0fd340599d686a020f187217_4b9a7812e88070d8f8c92ba7c915f7b9 store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:36,066 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209190869ad0fd340599d686a020f187217_4b9a7812e88070d8f8c92ba7c915f7b9, store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:36,066 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209190869ad0fd340599d686a020f187217_4b9a7812e88070d8f8c92ba7c915f7b9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:36,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742226_1402 (size=4469) 2024-12-09T01:42:36,073 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/6c9ceb64995049628f721870cefc7db0 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/6c9ceb64995049628f721870cefc7db0 2024-12-09T01:42:36,078 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b9a7812e88070d8f8c92ba7c915f7b9/B of 4b9a7812e88070d8f8c92ba7c915f7b9 into 6c9ceb64995049628f721870cefc7db0(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:36,078 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:36,078 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., storeName=4b9a7812e88070d8f8c92ba7c915f7b9/B, priority=12, startTime=1733708556040; duration=0sec 2024-12-09T01:42:36,078 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:36,078 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9a7812e88070d8f8c92ba7c915f7b9:B 2024-12-09T01:42:36,078 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:42:36,079 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:42:36,080 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 4b9a7812e88070d8f8c92ba7c915f7b9/C is initiating minor compaction (all files) 2024-12-09T01:42:36,080 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9a7812e88070d8f8c92ba7c915f7b9/C in TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:36,080 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/9d5ad637ce5c421f92dda57e61e61d27, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/f8812d756ab94cb391bcbc7a9cde01a0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/fb9e0a3ae23d49e8b00d74fe275c2a4f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/3da68982540943708c24e175ef37a065] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp, totalSize=47.0 K 2024-12-09T01:42:36,080 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9a7812e88070d8f8c92ba7c915f7b9#A#compaction#348 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:36,081 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d5ad637ce5c421f92dda57e61e61d27, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1733708552306 2024-12-09T01:42:36,082 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/5410b6ac536b4430906c30a0473290da is 175, key is test_row_0/A:col10/1733708555473/Put/seqid=0 2024-12-09T01:42:36,082 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting f8812d756ab94cb391bcbc7a9cde01a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1733708552464 2024-12-09T01:42:36,084 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting fb9e0a3ae23d49e8b00d74fe275c2a4f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1733708553171 2024-12-09T01:42:36,084 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 3da68982540943708c24e175ef37a065, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733708555472 2024-12-09T01:42:36,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742227_1403 (size=31195) 2024-12-09T01:42:36,096 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/5410b6ac536b4430906c30a0473290da as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/5410b6ac536b4430906c30a0473290da 2024-12-09T01:42:36,096 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9a7812e88070d8f8c92ba7c915f7b9#C#compaction#349 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:36,097 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/05d43f07dd1d46b4b7851225897b5a61 is 50, key is test_row_0/C:col10/1733708555473/Put/seqid=0 2024-12-09T01:42:36,103 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b9a7812e88070d8f8c92ba7c915f7b9/A of 4b9a7812e88070d8f8c92ba7c915f7b9 into 5410b6ac536b4430906c30a0473290da(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:36,103 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:36,103 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., storeName=4b9a7812e88070d8f8c92ba7c915f7b9/A, priority=12, startTime=1733708556040; duration=0sec 2024-12-09T01:42:36,103 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:36,103 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9a7812e88070d8f8c92ba7c915f7b9:A 2024-12-09T01:42:36,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742228_1404 (size=12241) 2024-12-09T01:42:36,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:36,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9a7812e88070d8f8c92ba7c915f7b9 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-12-09T01:42:36,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=A 2024-12-09T01:42:36,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:36,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=B 2024-12-09T01:42:36,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:36,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=C 2024-12-09T01:42:36,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:36,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120925e920c78de6452787b110a4fc7bf7b5_4b9a7812e88070d8f8c92ba7c915f7b9 is 50, key is test_row_0/A:col10/1733708556300/Put/seqid=0 2024-12-09T01:42:36,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-09T01:42:36,313 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:36,314 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:42:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-12-09T01:42:36,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T01:42:36,316 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:42:36,317 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:42:36,317 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:42:36,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742229_1405 (size=14794) 2024-12-09T01:42:36,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:36,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:36,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708616304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:36,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708616305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:36,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708616318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,322 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:36,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:36,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708616321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:36,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:36,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708616322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T01:42:36,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708616423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:36,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708616423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:36,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,432 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:36,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:36,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708616433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:36,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,437 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:36,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:36,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708616444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:36,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708616444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:36,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T01:42:36,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:36,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:36,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:36,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:36,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:36,516 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/05d43f07dd1d46b4b7851225897b5a61 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/05d43f07dd1d46b4b7851225897b5a61 2024-12-09T01:42:36,520 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b9a7812e88070d8f8c92ba7c915f7b9/C of 4b9a7812e88070d8f8c92ba7c915f7b9 into 05d43f07dd1d46b4b7851225897b5a61(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:36,520 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:36,521 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., storeName=4b9a7812e88070d8f8c92ba7c915f7b9/C, priority=12, startTime=1733708556040; duration=0sec 2024-12-09T01:42:36,521 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:36,521 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9a7812e88070d8f8c92ba7c915f7b9:C 2024-12-09T01:42:36,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T01:42:36,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:36,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:36,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:36,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:36,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T01:42:36,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708616644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:36,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708616644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:36,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708616644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:36,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:36,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:36,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708616653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:36,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:36,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708616654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:36,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:36,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,662 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:36,721 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:36,724 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120925e920c78de6452787b110a4fc7bf7b5_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120925e920c78de6452787b110a4fc7bf7b5_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:36,725 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/20695f4409b6469e9d8186595055ecc8, store: [table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:36,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/20695f4409b6469e9d8186595055ecc8 is 175, key is test_row_0/A:col10/1733708556300/Put/seqid=0 2024-12-09T01:42:36,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742230_1406 (size=39749) 2024-12-09T01:42:36,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T01:42:36,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:36,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:36,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:36,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:36,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:36,931 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T01:42:36,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:36,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:36,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:36,932 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:36,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:36,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:36,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T01:42:36,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:36,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708616964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:36,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,972 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708616964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:36,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:36,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:36,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708616964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:36,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:36,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,973 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:36,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:36,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708616974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:36,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:36,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708616974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:37,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T01:42:37,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:37,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:37,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:37,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:37,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:37,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:37,136 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/20695f4409b6469e9d8186595055ecc8 2024-12-09T01:42:37,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/bfb214fea9874901a6bec879a9a6ff36 is 50, key is test_row_0/B:col10/1733708556300/Put/seqid=0 2024-12-09T01:42:37,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742231_1407 (size=12151) 2024-12-09T01:42:37,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T01:42:37,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:37,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:37,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:37,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:37,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:37,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T01:42:37,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:37,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:37,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:37,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:37,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:37,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:37,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T01:42:37,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:37,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708617483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:37,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:37,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708617483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:37,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:37,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708617483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:37,489 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:37,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708617493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:37,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:37,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:37,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708617494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:37,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:37,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:37,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:37,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T01:42:37,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:37,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:37,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:37,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:37,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:37,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:37,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/bfb214fea9874901a6bec879a9a6ff36 2024-12-09T01:42:37,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/9edb812897a24b539bf687fe9110523d is 50, key is test_row_0/C:col10/1733708556300/Put/seqid=0 2024-12-09T01:42:37,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742232_1408 (size=12151) 2024-12-09T01:42:37,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/9edb812897a24b539bf687fe9110523d 2024-12-09T01:42:37,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/20695f4409b6469e9d8186595055ecc8 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/20695f4409b6469e9d8186595055ecc8 2024-12-09T01:42:37,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/20695f4409b6469e9d8186595055ecc8, entries=200, sequenceid=146, filesize=38.8 K 2024-12-09T01:42:37,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/bfb214fea9874901a6bec879a9a6ff36 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/bfb214fea9874901a6bec879a9a6ff36 2024-12-09T01:42:37,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/bfb214fea9874901a6bec879a9a6ff36, entries=150, sequenceid=146, filesize=11.9 K 2024-12-09T01:42:37,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/9edb812897a24b539bf687fe9110523d as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/9edb812897a24b539bf687fe9110523d 2024-12-09T01:42:37,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/9edb812897a24b539bf687fe9110523d, entries=150, sequenceid=146, filesize=11.9 K 2024-12-09T01:42:37,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~415.41 KB/425376, currentSize=33.54 KB/34350 for 4b9a7812e88070d8f8c92ba7c915f7b9 in 1289ms, sequenceid=146, compaction requested=false 2024-12-09T01:42:37,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:37,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-12-09T01:42:37,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:37,700 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 4b9a7812e88070d8f8c92ba7c915f7b9 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-12-09T01:42:37,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=A 2024-12-09T01:42:37,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:37,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=B 2024-12-09T01:42:37,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:37,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=C 2024-12-09T01:42:37,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:37,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ff99d5ae42a84238ab8438aa70d35bd2_4b9a7812e88070d8f8c92ba7c915f7b9 is 50, key is test_row_0/A:col10/1733708556317/Put/seqid=0 2024-12-09T01:42:37,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742233_1409 (size=12304) 2024-12-09T01:42:38,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:38,130 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209ff99d5ae42a84238ab8438aa70d35bd2_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ff99d5ae42a84238ab8438aa70d35bd2_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:38,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/3eb534ed50cb451590ecfc9f7b6a0abc, store: [table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:38,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/3eb534ed50cb451590ecfc9f7b6a0abc is 175, key is test_row_0/A:col10/1733708556317/Put/seqid=0 2024-12-09T01:42:38,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742234_1410 (size=31105) 2024-12-09T01:42:38,144 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/3eb534ed50cb451590ecfc9f7b6a0abc 2024-12-09T01:42:38,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/3f0d642b8f824791abd4b82b30e9ebc2 is 50, key is test_row_0/B:col10/1733708556317/Put/seqid=0 2024-12-09T01:42:38,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742235_1411 (size=12151) 2024-12-09T01:42:38,187 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/3f0d642b8f824791abd4b82b30e9ebc2 2024-12-09T01:42:38,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/7ee4ec2078944a91bfc9e053ba6d9f96 is 50, key is test_row_0/C:col10/1733708556317/Put/seqid=0 2024-12-09T01:42:38,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742236_1412 (size=12151) 2024-12-09T01:42:38,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T01:42:38,504 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:38,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:38,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733708558516, max timestamp=1733708558556Num uniques 859; 2024-12-09T01:42:38,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733708558516, max timestamp=1733708558556Num uniques 859; 2024-12-09T01:42:38,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-09T01:42:38,559 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-09T01:42:38,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1733708558516, max timestamp=1733708558556Num uniques 859; 2024-12-09T01:42:38,560 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-12-09T01:42:38,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:38,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708618595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:38,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,602 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:38,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708618596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:38,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:38,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708618597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:38,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:38,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:38,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708618598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:38,615 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:38,615 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708618599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:38,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:38,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,615 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:38,617 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/7ee4ec2078944a91bfc9e053ba6d9f96 2024-12-09T01:42:38,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/3eb534ed50cb451590ecfc9f7b6a0abc as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3eb534ed50cb451590ecfc9f7b6a0abc 2024-12-09T01:42:38,628 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3eb534ed50cb451590ecfc9f7b6a0abc, entries=150, sequenceid=155, filesize=30.4 K 2024-12-09T01:42:38,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/3f0d642b8f824791abd4b82b30e9ebc2 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/3f0d642b8f824791abd4b82b30e9ebc2 2024-12-09T01:42:38,633 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/3f0d642b8f824791abd4b82b30e9ebc2, entries=150, sequenceid=155, filesize=11.9 K 2024-12-09T01:42:38,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/7ee4ec2078944a91bfc9e053ba6d9f96 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/7ee4ec2078944a91bfc9e053ba6d9f96 2024-12-09T01:42:38,637 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/7ee4ec2078944a91bfc9e053ba6d9f96, entries=150, sequenceid=155, filesize=11.9 K 2024-12-09T01:42:38,638 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for 4b9a7812e88070d8f8c92ba7c915f7b9 in 939ms, sequenceid=155, compaction requested=true 2024-12-09T01:42:38,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:38,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:38,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-12-09T01:42:38,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-12-09T01:42:38,642 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-12-09T01:42:38,642 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3230 sec 2024-12-09T01:42:38,643 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.3280 sec 2024-12-09T01:42:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:38,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9a7812e88070d8f8c92ba7c915f7b9 3/3 column families, dataSize=161.02 KB heapSize=445.92 KB 2024-12-09T01:42:38,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=A 2024-12-09T01:42:38,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:38,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=B 2024-12-09T01:42:38,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:38,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=C 2024-12-09T01:42:38,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:38,728 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120965a4a9fb16514c109e1eb3328d8abe1f_4b9a7812e88070d8f8c92ba7c915f7b9 is 50, key is test_row_0/A:col10/1733708558598/Put/seqid=0 2024-12-09T01:42:38,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742237_1413 (size=14794) 2024-12-09T01:42:38,733 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:38,736 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120965a4a9fb16514c109e1eb3328d8abe1f_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120965a4a9fb16514c109e1eb3328d8abe1f_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:38,737 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/07222b09e39448d68010faa1d88d8ac3, store: [table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:38,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/07222b09e39448d68010faa1d88d8ac3 is 175, key is test_row_0/A:col10/1733708558598/Put/seqid=0 2024-12-09T01:42:38,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:38,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708618724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:38,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742238_1414 (size=39749) 2024-12-09T01:42:38,744 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=192, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/07222b09e39448d68010faa1d88d8ac3 2024-12-09T01:42:38,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:38,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708618736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:38,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:38,745 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708618736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:38,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/5b091605999a4090b3791f81677fd963 is 50, key is test_row_0/B:col10/1733708558598/Put/seqid=0 2024-12-09T01:42:38,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:38,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708618742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708618740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:38,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,758 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:38,758 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:38,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742239_1415 (size=12151) 2024-12-09T01:42:38,761 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/5b091605999a4090b3791f81677fd963 2024-12-09T01:42:38,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/076e125e2be84f7bb5834c8cc65772bb is 50, key is test_row_0/C:col10/1733708558598/Put/seqid=0 2024-12-09T01:42:38,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742240_1416 (size=12151) 2024-12-09T01:42:38,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708618853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:38,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708618853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:38,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708618864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:38,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708618864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:38,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:38,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708618954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:38,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:38,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:39,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708619064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:39,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708619064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:39,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,075 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:39,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:39,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708619083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:39,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:39,088 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708619083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:39,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:39,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:39,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/076e125e2be84f7bb5834c8cc65772bb 2024-12-09T01:42:39,182 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/07222b09e39448d68010faa1d88d8ac3 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/07222b09e39448d68010faa1d88d8ac3 2024-12-09T01:42:39,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/07222b09e39448d68010faa1d88d8ac3, entries=200, sequenceid=192, filesize=38.8 K 2024-12-09T01:42:39,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/5b091605999a4090b3791f81677fd963 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/5b091605999a4090b3791f81677fd963 2024-12-09T01:42:39,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/5b091605999a4090b3791f81677fd963, entries=150, sequenceid=192, filesize=11.9 K 2024-12-09T01:42:39,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/076e125e2be84f7bb5834c8cc65772bb as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/076e125e2be84f7bb5834c8cc65772bb 2024-12-09T01:42:39,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/076e125e2be84f7bb5834c8cc65772bb, entries=150, sequenceid=192, filesize=11.9 K 2024-12-09T01:42:39,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~445.88 KB/456576, currentSize=26.84 KB/27480 for 4b9a7812e88070d8f8c92ba7c915f7b9 in 477ms, sequenceid=192, compaction requested=true 2024-12-09T01:42:39,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:39,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9a7812e88070d8f8c92ba7c915f7b9:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:42:39,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:39,197 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:42:39,197 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:42:39,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9a7812e88070d8f8c92ba7c915f7b9:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:42:39,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:39,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9a7812e88070d8f8c92ba7c915f7b9:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:42:39,198 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:39,198 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141798 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:42:39,198 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:42:39,198 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 4b9a7812e88070d8f8c92ba7c915f7b9/A is initiating minor compaction (all files) 2024-12-09T01:42:39,198 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 4b9a7812e88070d8f8c92ba7c915f7b9/B is initiating minor compaction (all files) 2024-12-09T01:42:39,199 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9a7812e88070d8f8c92ba7c915f7b9/A in TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:39,199 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9a7812e88070d8f8c92ba7c915f7b9/B in TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:39,199 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/5410b6ac536b4430906c30a0473290da, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/20695f4409b6469e9d8186595055ecc8, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3eb534ed50cb451590ecfc9f7b6a0abc, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/07222b09e39448d68010faa1d88d8ac3] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp, totalSize=138.5 K 2024-12-09T01:42:39,199 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/6c9ceb64995049628f721870cefc7db0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/bfb214fea9874901a6bec879a9a6ff36, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/3f0d642b8f824791abd4b82b30e9ebc2, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/5b091605999a4090b3791f81677fd963] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp, totalSize=47.6 K 2024-12-09T01:42:39,199 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:39,199 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. files: [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/5410b6ac536b4430906c30a0473290da, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/20695f4409b6469e9d8186595055ecc8, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3eb534ed50cb451590ecfc9f7b6a0abc, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/07222b09e39448d68010faa1d88d8ac3] 2024-12-09T01:42:39,199 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c9ceb64995049628f721870cefc7db0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733708555472 2024-12-09T01:42:39,199 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5410b6ac536b4430906c30a0473290da, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733708555472 2024-12-09T01:42:39,199 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20695f4409b6469e9d8186595055ecc8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733708555625 2024-12-09T01:42:39,199 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting bfb214fea9874901a6bec879a9a6ff36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733708555625 2024-12-09T01:42:39,200 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eb534ed50cb451590ecfc9f7b6a0abc, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733708556316 2024-12-09T01:42:39,200 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f0d642b8f824791abd4b82b30e9ebc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733708556316 2024-12-09T01:42:39,200 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07222b09e39448d68010faa1d88d8ac3, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733708558576 2024-12-09T01:42:39,200 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b091605999a4090b3791f81677fd963, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733708558576 2024-12-09T01:42:39,210 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:39,211 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9a7812e88070d8f8c92ba7c915f7b9#B#compaction#359 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:39,211 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/e7a7cfd17ecb459c96bedbe9ba7c2113 is 50, key is test_row_0/B:col10/1733708558598/Put/seqid=0 2024-12-09T01:42:39,224 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209c6c6053d2a3748d89399f1af48ac5cdf_4b9a7812e88070d8f8c92ba7c915f7b9 store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:39,227 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209c6c6053d2a3748d89399f1af48ac5cdf_4b9a7812e88070d8f8c92ba7c915f7b9, store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:39,227 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c6c6053d2a3748d89399f1af48ac5cdf_4b9a7812e88070d8f8c92ba7c915f7b9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:39,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742241_1417 (size=12527) 2024-12-09T01:42:39,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742242_1418 (size=4469) 2024-12-09T01:42:39,256 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9a7812e88070d8f8c92ba7c915f7b9#A#compaction#360 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:39,257 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/16d3e1ccc55646a69698fe5f7e844ca0 is 175, key is test_row_0/A:col10/1733708558598/Put/seqid=0 2024-12-09T01:42:39,265 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/e7a7cfd17ecb459c96bedbe9ba7c2113 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/e7a7cfd17ecb459c96bedbe9ba7c2113 2024-12-09T01:42:39,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742243_1419 (size=31481) 2024-12-09T01:42:39,273 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b9a7812e88070d8f8c92ba7c915f7b9/B of 4b9a7812e88070d8f8c92ba7c915f7b9 into e7a7cfd17ecb459c96bedbe9ba7c2113(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:39,273 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/16d3e1ccc55646a69698fe5f7e844ca0 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/16d3e1ccc55646a69698fe5f7e844ca0 2024-12-09T01:42:39,274 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:39,274 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., storeName=4b9a7812e88070d8f8c92ba7c915f7b9/B, priority=12, startTime=1733708559197; duration=0sec 2024-12-09T01:42:39,274 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:39,274 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9a7812e88070d8f8c92ba7c915f7b9:B 2024-12-09T01:42:39,274 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:42:39,277 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:42:39,277 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 4b9a7812e88070d8f8c92ba7c915f7b9/C is initiating minor compaction (all files) 2024-12-09T01:42:39,277 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9a7812e88070d8f8c92ba7c915f7b9/C in TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:39,278 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/05d43f07dd1d46b4b7851225897b5a61, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/9edb812897a24b539bf687fe9110523d, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/7ee4ec2078944a91bfc9e053ba6d9f96, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/076e125e2be84f7bb5834c8cc65772bb] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp, totalSize=47.6 K 2024-12-09T01:42:39,278 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 05d43f07dd1d46b4b7851225897b5a61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733708555472 2024-12-09T01:42:39,279 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 9edb812897a24b539bf687fe9110523d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1733708555625 2024-12-09T01:42:39,279 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ee4ec2078944a91bfc9e053ba6d9f96, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1733708556316 2024-12-09T01:42:39,280 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 076e125e2be84f7bb5834c8cc65772bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733708558576 2024-12-09T01:42:39,281 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b9a7812e88070d8f8c92ba7c915f7b9/A of 4b9a7812e88070d8f8c92ba7c915f7b9 into 16d3e1ccc55646a69698fe5f7e844ca0(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:39,281 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:39,281 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., storeName=4b9a7812e88070d8f8c92ba7c915f7b9/A, priority=12, startTime=1733708559197; duration=0sec 2024-12-09T01:42:39,281 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:39,281 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9a7812e88070d8f8c92ba7c915f7b9:A 2024-12-09T01:42:39,291 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9a7812e88070d8f8c92ba7c915f7b9#C#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:39,291 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/fc7be14357614621a0083f69123cbdab is 50, key is test_row_0/C:col10/1733708558598/Put/seqid=0 2024-12-09T01:42:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:39,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9a7812e88070d8f8c92ba7c915f7b9 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:42:39,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=A 2024-12-09T01:42:39,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:39,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=B 2024-12-09T01:42:39,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:39,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=C 2024-12-09T01:42:39,293 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:39,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742244_1420 (size=12527) 2024-12-09T01:42:39,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120921ea7c7a5d30417993b57efcf6670be7_4b9a7812e88070d8f8c92ba7c915f7b9 is 50, key is test_row_0/A:col10/1733708558735/Put/seqid=0 2024-12-09T01:42:39,308 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/fc7be14357614621a0083f69123cbdab as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/fc7be14357614621a0083f69123cbdab 2024-12-09T01:42:39,312 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b9a7812e88070d8f8c92ba7c915f7b9/C of 4b9a7812e88070d8f8c92ba7c915f7b9 into fc7be14357614621a0083f69123cbdab(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:39,312 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:39,312 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., storeName=4b9a7812e88070d8f8c92ba7c915f7b9/C, priority=12, startTime=1733708559198; duration=0sec 2024-12-09T01:42:39,312 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:39,312 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9a7812e88070d8f8c92ba7c915f7b9:C 2024-12-09T01:42:39,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742245_1421 (size=14794) 2024-12-09T01:42:39,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708559436, max timestamp=1733708559451Num uniques 859; 2024-12-09T01:42:39,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708559436, max timestamp=1733708559451Num uniques 859; 2024-12-09T01:42:39,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:42:39,455 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:42:39,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708559436, max timestamp=1733708559451Num uniques 859; 2024-12-09T01:42:39,456 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:42:39,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:39,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708619494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708619492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,515 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:39,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708619495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:39,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:39,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708619508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:39,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:39,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708619515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:39,636 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708619623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:39,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708619623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:39,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:39,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708619624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:39,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:39,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708619634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:39,638 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:39,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:39,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708619634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:39,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:39,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:39,717 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:39,721 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120921ea7c7a5d30417993b57efcf6670be7_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120921ea7c7a5d30417993b57efcf6670be7_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:39,721 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/52e51917b5a2412eb17e7259ebaaeabb, store: [table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:39,722 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/52e51917b5a2412eb17e7259ebaaeabb is 175, key is test_row_0/A:col10/1733708558735/Put/seqid=0 2024-12-09T01:42:39,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742246_1422 (size=39749) 2024-12-09T01:42:39,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708619844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:39,855 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:39,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708619844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:39,855 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:39,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708619844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708619844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708619844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:39,856 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:39,857 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:40,127 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/52e51917b5a2412eb17e7259ebaaeabb 2024-12-09T01:42:40,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/543c721de2c5461aa233dd73225c2b53 is 50, key is test_row_0/B:col10/1733708558735/Put/seqid=0 2024-12-09T01:42:40,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742247_1423 (size=12151) 2024-12-09T01:42:40,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708620164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:40,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708620164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:40,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:40,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:40,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708620164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:40,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:40,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708620164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:40,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708620164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:40,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,176 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:40,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-09T01:42:40,484 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:40,484 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:42:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-12-09T01:42:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T01:42:40,487 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:42:40,487 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:42:40,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:42:40,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/543c721de2c5461aa233dd73225c2b53 2024-12-09T01:42:40,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/db846a40393b4f40bd543bcfa312c81a is 50, key is test_row_0/C:col10/1733708558735/Put/seqid=0 2024-12-09T01:42:40,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742248_1424 (size=12151) 2024-12-09T01:42:40,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/db846a40393b4f40bd543bcfa312c81a 2024-12-09T01:42:40,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/52e51917b5a2412eb17e7259ebaaeabb as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/52e51917b5a2412eb17e7259ebaaeabb 2024-12-09T01:42:40,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/52e51917b5a2412eb17e7259ebaaeabb, entries=200, sequenceid=204, filesize=38.8 K 2024-12-09T01:42:40,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/543c721de2c5461aa233dd73225c2b53 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/543c721de2c5461aa233dd73225c2b53 2024-12-09T01:42:40,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/543c721de2c5461aa233dd73225c2b53, entries=150, sequenceid=204, filesize=11.9 K 2024-12-09T01:42:40,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/db846a40393b4f40bd543bcfa312c81a as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/db846a40393b4f40bd543bcfa312c81a 2024-12-09T01:42:40,575 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/db846a40393b4f40bd543bcfa312c81a, entries=150, sequenceid=204, filesize=11.9 K 2024-12-09T01:42:40,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 4b9a7812e88070d8f8c92ba7c915f7b9 in 1285ms, sequenceid=204, compaction requested=false 2024-12-09T01:42:40,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:40,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T01:42:40,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-12-09T01:42:40,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:40,641 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 4b9a7812e88070d8f8c92ba7c915f7b9 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-12-09T01:42:40,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=A 2024-12-09T01:42:40,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:40,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=B 2024-12-09T01:42:40,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:40,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=C 2024-12-09T01:42:40,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:40,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120912710fd3425a4ee6ac3fc871aec384ab_4b9a7812e88070d8f8c92ba7c915f7b9 is 50, key is test_row_0/A:col10/1733708559513/Put/seqid=0 2024-12-09T01:42:40,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742249_1425 (size=12304) 2024-12-09T01:42:40,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:40,661 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120912710fd3425a4ee6ac3fc871aec384ab_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120912710fd3425a4ee6ac3fc871aec384ab_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:40,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/68e683e0f4fc47eea62b3d1c61820343, store: [table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:40,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/68e683e0f4fc47eea62b3d1c61820343 is 175, key is test_row_0/A:col10/1733708559513/Put/seqid=0 2024-12-09T01:42:40,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742250_1426 (size=31105) 2024-12-09T01:42:40,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:40,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:40,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:40,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708620696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:40,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:40,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708620698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:40,716 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,716 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:40,717 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:40,717 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:40,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708620703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:40,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:40,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:40,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708620716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:40,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:40,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708620717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:40,732 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:40,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:40,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T01:42:40,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:40,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708620823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:40,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:40,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708620824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:40,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,832 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:40,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:40,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708620824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:40,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:40,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,832 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:40,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708620843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:40,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708620843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:40,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:40,857 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:41,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:41,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708621044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:41,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708621044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:41,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708621044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:41,070 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=242, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/68e683e0f4fc47eea62b3d1c61820343 2024-12-09T01:42:41,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708621063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:41,078 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708621063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:41,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/20a1cc038125472c8ab5a0551d78915b is 50, key is test_row_0/B:col10/1733708559513/Put/seqid=0 2024-12-09T01:42:41,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742251_1427 (size=12151) 2024-12-09T01:42:41,085 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/20a1cc038125472c8ab5a0551d78915b 2024-12-09T01:42:41,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/b97e4bdb7e2148d79851cf939c9c32e6 is 50, key is test_row_0/C:col10/1733708559513/Put/seqid=0 2024-12-09T01:42:41,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742252_1428 (size=12151) 2024-12-09T01:42:41,109 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=242 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/b97e4bdb7e2148d79851cf939c9c32e6 2024-12-09T01:42:41,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T01:42:41,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/68e683e0f4fc47eea62b3d1c61820343 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/68e683e0f4fc47eea62b3d1c61820343 2024-12-09T01:42:41,117 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/68e683e0f4fc47eea62b3d1c61820343, entries=150, sequenceid=242, filesize=30.4 K 2024-12-09T01:42:41,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/20a1cc038125472c8ab5a0551d78915b as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/20a1cc038125472c8ab5a0551d78915b 2024-12-09T01:42:41,122 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/20a1cc038125472c8ab5a0551d78915b, entries=150, sequenceid=242, filesize=11.9 K 2024-12-09T01:42:41,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/b97e4bdb7e2148d79851cf939c9c32e6 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/b97e4bdb7e2148d79851cf939c9c32e6 2024-12-09T01:42:41,126 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/b97e4bdb7e2148d79851cf939c9c32e6, entries=150, sequenceid=242, filesize=11.9 K 2024-12-09T01:42:41,127 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for 4b9a7812e88070d8f8c92ba7c915f7b9 in 487ms, sequenceid=242, compaction requested=true 2024-12-09T01:42:41,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:41,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:41,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-12-09T01:42:41,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-12-09T01:42:41,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-12-09T01:42:41,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 640 msec 2024-12-09T01:42:41,132 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 645 msec 2024-12-09T01:42:41,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:41,370 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9a7812e88070d8f8c92ba7c915f7b9 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T01:42:41,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=A 2024-12-09T01:42:41,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:41,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=B 2024-12-09T01:42:41,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:41,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=C 2024-12-09T01:42:41,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:41,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120937932c0ddee1445c9f0b0accd3eff474_4b9a7812e88070d8f8c92ba7c915f7b9 is 50, key is test_row_0/A:col10/1733708560715/Put/seqid=0 2024-12-09T01:42:41,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742253_1429 (size=14794) 2024-12-09T01:42:41,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708621419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:41,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708621421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:41,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:41,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708621425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:41,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:41,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708621427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:41,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,442 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:41,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:41,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708621437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:41,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:41,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708621543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:41,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708621544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,556 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:41,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708621544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708621544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,557 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:41,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708621553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:41,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:41,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-12-09T01:42:41,624 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:41,625 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:42:41,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-12-09T01:42:41,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T01:42:41,626 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:42:41,627 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:42:41,627 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:42:41,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T01:42:41,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:41,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708621763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:41,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:41,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708621763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:41,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,772 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:41,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:41,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708621763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:41,772 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:41,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:41,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:41,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708621764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:41,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:41,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708621773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:41,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:41,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:41,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-09T01:42:41,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:41,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:41,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:41,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:41,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:41,791 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:41,795 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120937932c0ddee1445c9f0b0accd3eff474_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120937932c0ddee1445c9f0b0accd3eff474_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:41,796 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/00f1a68da3ef4f94a54bd6d50c346450, store: [table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:41,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/00f1a68da3ef4f94a54bd6d50c346450 is 175, key is test_row_0/A:col10/1733708560715/Put/seqid=0 2024-12-09T01:42:41,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742254_1430 (size=39749) 2024-12-09T01:42:41,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-09T01:42:41,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:41,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:41,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:41,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:41,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:41,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:41,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T01:42:42,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-09T01:42:42,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:42,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:42,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:42,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:42,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:42,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:42,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708622083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:42,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:42,096 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708622084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:42,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708622084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:42,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708622084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:42,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:42,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708622084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:42,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:42,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:42,098 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:42,201 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/00f1a68da3ef4f94a54bd6d50c346450 2024-12-09T01:42:42,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/630e4f93ccee45759a647d6bd4311dec is 50, key is test_row_0/B:col10/1733708560715/Put/seqid=0 2024-12-09T01:42:42,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742255_1431 (size=12151) 2024-12-09T01:42:42,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-09T01:42:42,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:42,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:42,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:42,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:42,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:42,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T01:42:42,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-09T01:42:42,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:42,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:42,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:42,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:42,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:42,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:42,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-09T01:42:42,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:42,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:42,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:42,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:42,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:42,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:42,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708622604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:42,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:42,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:42,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708622604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:42,613 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:42,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708622604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:42,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:42,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708622604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:42,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:42,614 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:42,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708622604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:42,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:42,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:42,615 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:42,619 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/630e4f93ccee45759a647d6bd4311dec 2024-12-09T01:42:42,627 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/ff35c79335e94125a7d157f38205fa1a is 50, key is test_row_0/C:col10/1733708560715/Put/seqid=0 2024-12-09T01:42:42,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742256_1432 (size=12151) 2024-12-09T01:42:42,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/ff35c79335e94125a7d157f38205fa1a 2024-12-09T01:42:42,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/00f1a68da3ef4f94a54bd6d50c346450 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/00f1a68da3ef4f94a54bd6d50c346450 2024-12-09T01:42:42,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/00f1a68da3ef4f94a54bd6d50c346450, entries=200, sequenceid=254, filesize=38.8 K 2024-12-09T01:42:42,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/630e4f93ccee45759a647d6bd4311dec as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/630e4f93ccee45759a647d6bd4311dec 2024-12-09T01:42:42,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/630e4f93ccee45759a647d6bd4311dec, entries=150, sequenceid=254, filesize=11.9 K 2024-12-09T01:42:42,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/ff35c79335e94125a7d157f38205fa1a as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/ff35c79335e94125a7d157f38205fa1a 2024-12-09T01:42:42,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/ff35c79335e94125a7d157f38205fa1a, entries=150, sequenceid=254, filesize=11.9 K 2024-12-09T01:42:42,660 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 4b9a7812e88070d8f8c92ba7c915f7b9 in 1290ms, sequenceid=254, compaction requested=true 2024-12-09T01:42:42,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:42,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9a7812e88070d8f8c92ba7c915f7b9:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:42:42,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:42,661 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:42:42,661 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:42:42,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9a7812e88070d8f8c92ba7c915f7b9:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:42:42,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:42,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9a7812e88070d8f8c92ba7c915f7b9:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:42:42,661 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:42,663 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:42:42,663 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 4b9a7812e88070d8f8c92ba7c915f7b9/B is initiating minor compaction (all files) 2024-12-09T01:42:42,663 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9a7812e88070d8f8c92ba7c915f7b9/B in TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:42,663 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/e7a7cfd17ecb459c96bedbe9ba7c2113, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/543c721de2c5461aa233dd73225c2b53, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/20a1cc038125472c8ab5a0551d78915b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/630e4f93ccee45759a647d6bd4311dec] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp, totalSize=47.8 K 2024-12-09T01:42:42,664 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142084 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:42:42,664 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 4b9a7812e88070d8f8c92ba7c915f7b9/A is initiating minor compaction (all files) 2024-12-09T01:42:42,664 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9a7812e88070d8f8c92ba7c915f7b9/A in TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:42,664 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/16d3e1ccc55646a69698fe5f7e844ca0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/52e51917b5a2412eb17e7259ebaaeabb, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/68e683e0f4fc47eea62b3d1c61820343, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/00f1a68da3ef4f94a54bd6d50c346450] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp, totalSize=138.8 K 2024-12-09T01:42:42,664 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:42,664 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. files: [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/16d3e1ccc55646a69698fe5f7e844ca0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/52e51917b5a2412eb17e7259ebaaeabb, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/68e683e0f4fc47eea62b3d1c61820343, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/00f1a68da3ef4f94a54bd6d50c346450] 2024-12-09T01:42:42,664 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting e7a7cfd17ecb459c96bedbe9ba7c2113, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733708558576 2024-12-09T01:42:42,665 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16d3e1ccc55646a69698fe5f7e844ca0, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733708558576 2024-12-09T01:42:42,665 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 543c721de2c5461aa233dd73225c2b53, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733708558735 2024-12-09T01:42:42,665 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52e51917b5a2412eb17e7259ebaaeabb, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733708558735 2024-12-09T01:42:42,667 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68e683e0f4fc47eea62b3d1c61820343, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733708559490 2024-12-09T01:42:42,667 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 20a1cc038125472c8ab5a0551d78915b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733708559490 2024-12-09T01:42:42,668 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00f1a68da3ef4f94a54bd6d50c346450, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733708560693 2024-12-09T01:42:42,668 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 630e4f93ccee45759a647d6bd4311dec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733708560714 2024-12-09T01:42:42,676 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:42,680 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412096a3bb5bf2f6f49b1a586ecc68ea44630_4b9a7812e88070d8f8c92ba7c915f7b9 store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:42,682 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9a7812e88070d8f8c92ba7c915f7b9#B#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:42,682 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/b00b50c735504e4ab0c758cb13531e85 is 50, key is test_row_0/B:col10/1733708560715/Put/seqid=0 2024-12-09T01:42:42,684 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412096a3bb5bf2f6f49b1a586ecc68ea44630_4b9a7812e88070d8f8c92ba7c915f7b9, store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:42,684 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096a3bb5bf2f6f49b1a586ecc68ea44630_4b9a7812e88070d8f8c92ba7c915f7b9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:42,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742258_1434 (size=12663) 2024-12-09T01:42:42,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742257_1433 (size=4469) 2024-12-09T01:42:42,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-12-09T01:42:42,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:42,700 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 4b9a7812e88070d8f8c92ba7c915f7b9 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T01:42:42,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=A 2024-12-09T01:42:42,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:42,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=B 2024-12-09T01:42:42,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:42,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=C 2024-12-09T01:42:42,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:42,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209910c02e62a3a44b19e1e7d484d2d417c_4b9a7812e88070d8f8c92ba7c915f7b9 is 50, key is test_row_0/A:col10/1733708561435/Put/seqid=0 2024-12-09T01:42:42,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742259_1435 (size=12454) 2024-12-09T01:42:42,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T01:42:43,103 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9a7812e88070d8f8c92ba7c915f7b9#A#compaction#371 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:43,103 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/348f59da2afa48d2ac9f19718f745f45 is 175, key is test_row_0/A:col10/1733708560715/Put/seqid=0 2024-12-09T01:42:43,112 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/b00b50c735504e4ab0c758cb13531e85 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/b00b50c735504e4ab0c758cb13531e85 2024-12-09T01:42:43,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:43,132 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b9a7812e88070d8f8c92ba7c915f7b9/B of 4b9a7812e88070d8f8c92ba7c915f7b9 into b00b50c735504e4ab0c758cb13531e85(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:43,132 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:43,132 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., storeName=4b9a7812e88070d8f8c92ba7c915f7b9/B, priority=12, startTime=1733708562661; duration=0sec 2024-12-09T01:42:43,132 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:43,132 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9a7812e88070d8f8c92ba7c915f7b9:B 2024-12-09T01:42:43,132 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:42:43,134 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:42:43,135 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 4b9a7812e88070d8f8c92ba7c915f7b9/C is initiating minor compaction (all files) 2024-12-09T01:42:43,135 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9a7812e88070d8f8c92ba7c915f7b9/C in TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:43,135 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/fc7be14357614621a0083f69123cbdab, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/db846a40393b4f40bd543bcfa312c81a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/b97e4bdb7e2148d79851cf939c9c32e6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/ff35c79335e94125a7d157f38205fa1a] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp, totalSize=47.8 K 2024-12-09T01:42:43,140 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209910c02e62a3a44b19e1e7d484d2d417c_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209910c02e62a3a44b19e1e7d484d2d417c_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:43,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/187b947dbdd84a698c61adeb609e2aae, store: [table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:43,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/187b947dbdd84a698c61adeb609e2aae is 175, key is test_row_0/A:col10/1733708561435/Put/seqid=0 2024-12-09T01:42:43,143 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting fc7be14357614621a0083f69123cbdab, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1733708558576 2024-12-09T01:42:43,148 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting db846a40393b4f40bd543bcfa312c81a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1733708558735 2024-12-09T01:42:43,149 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting b97e4bdb7e2148d79851cf939c9c32e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=242, earliestPutTs=1733708559490 2024-12-09T01:42:43,158 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting ff35c79335e94125a7d157f38205fa1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733708560714 2024-12-09T01:42:43,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742260_1436 (size=31617) 2024-12-09T01:42:43,171 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/348f59da2afa48d2ac9f19718f745f45 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/348f59da2afa48d2ac9f19718f745f45 2024-12-09T01:42:43,179 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b9a7812e88070d8f8c92ba7c915f7b9/A of 4b9a7812e88070d8f8c92ba7c915f7b9 into 348f59da2afa48d2ac9f19718f745f45(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:43,179 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:43,179 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., storeName=4b9a7812e88070d8f8c92ba7c915f7b9/A, priority=12, startTime=1733708562661; duration=0sec 2024-12-09T01:42:43,179 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:43,179 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9a7812e88070d8f8c92ba7c915f7b9:A 2024-12-09T01:42:43,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742261_1437 (size=31255) 2024-12-09T01:42:43,187 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/187b947dbdd84a698c61adeb609e2aae 2024-12-09T01:42:43,202 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9a7812e88070d8f8c92ba7c915f7b9#C#compaction#374 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:43,203 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/d6a74488203b404da1c04836b7ab8e16 is 50, key is test_row_0/C:col10/1733708560715/Put/seqid=0 2024-12-09T01:42:43,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/128e3f7424af48a390926a071fe373d1 is 50, key is test_row_0/B:col10/1733708561435/Put/seqid=0 2024-12-09T01:42:43,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742262_1438 (size=12663) 2024-12-09T01:42:43,265 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/d6a74488203b404da1c04836b7ab8e16 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/d6a74488203b404da1c04836b7ab8e16 2024-12-09T01:42:43,279 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4b9a7812e88070d8f8c92ba7c915f7b9/C of 4b9a7812e88070d8f8c92ba7c915f7b9 into d6a74488203b404da1c04836b7ab8e16(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:43,280 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:43,280 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., storeName=4b9a7812e88070d8f8c92ba7c915f7b9/C, priority=12, startTime=1733708562661; duration=0sec 2024-12-09T01:42:43,280 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:43,280 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9a7812e88070d8f8c92ba7c915f7b9:C 2024-12-09T01:42:43,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742263_1439 (size=12301) 2024-12-09T01:42:43,293 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/128e3f7424af48a390926a071fe373d1 2024-12-09T01:42:43,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/b6e9c15aae864bf7be295c622196308d is 50, key is test_row_0/C:col10/1733708561435/Put/seqid=0 2024-12-09T01:42:43,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742264_1440 (size=12301) 2024-12-09T01:42:43,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:43,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:43,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:43,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708623641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:43,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:43,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:43,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:43,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708623642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:43,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:43,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708623644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:43,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708623643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:43,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:43,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708623644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:43,754 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/b6e9c15aae864bf7be295c622196308d 2024-12-09T01:42:43,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:43,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708623753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:43,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/187b947dbdd84a698c61adeb609e2aae as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/187b947dbdd84a698c61adeb609e2aae 2024-12-09T01:42:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:43,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:43,770 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/187b947dbdd84a698c61adeb609e2aae, entries=150, sequenceid=274, filesize=30.5 K 2024-12-09T01:42:43,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/128e3f7424af48a390926a071fe373d1 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/128e3f7424af48a390926a071fe373d1 2024-12-09T01:42:43,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708623764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:43,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:43,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T01:42:43,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:43,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:43,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:43,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708623764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:43,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:43,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:43,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:43,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708623765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:43,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:43,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708623765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:43,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:43,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:43,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:43,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:43,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:43,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:43,777 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/128e3f7424af48a390926a071fe373d1, entries=150, sequenceid=274, filesize=12.0 K 2024-12-09T01:42:43,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/b6e9c15aae864bf7be295c622196308d as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/b6e9c15aae864bf7be295c622196308d 2024-12-09T01:42:43,783 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/b6e9c15aae864bf7be295c622196308d, entries=150, sequenceid=274, filesize=12.0 K 2024-12-09T01:42:43,784 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 4b9a7812e88070d8f8c92ba7c915f7b9 in 1084ms, sequenceid=274, compaction requested=false 2024-12-09T01:42:43,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:43,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:43,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-12-09T01:42:43,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-12-09T01:42:43,788 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-12-09T01:42:43,788 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1590 sec 2024-12-09T01:42:43,792 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.1650 sec 2024-12-09T01:42:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:43,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9a7812e88070d8f8c92ba7c915f7b9 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T01:42:43,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=A 2024-12-09T01:42:43,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:43,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=B 2024-12-09T01:42:43,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:43,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=C 2024-12-09T01:42:43,977 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:43,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097a3728f3593040f1add0b7491fa4af28_4b9a7812e88070d8f8c92ba7c915f7b9 is 50, key is test_row_0/A:col10/1733708563975/Put/seqid=0 2024-12-09T01:42:43,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742265_1441 (size=14994) 2024-12-09T01:42:43,998 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:44,003 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097a3728f3593040f1add0b7491fa4af28_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097a3728f3593040f1add0b7491fa4af28_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:44,004 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/28ef0ee2573042b68181cdf6b7ee1958, store: [table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:44,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/28ef0ee2573042b68181cdf6b7ee1958 is 175, key is test_row_0/A:col10/1733708563975/Put/seqid=0 2024-12-09T01:42:44,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742266_1442 (size=39949) 2024-12-09T01:42:44,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:44,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708624006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:44,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708624012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:44,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:44,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708624013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:44,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:44,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:44,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708624014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:44,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:44,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708624015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:44,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:44,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708624124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:44,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:44,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708624124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:44,135 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:44,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:44,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,135 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708624124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:44,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:44,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708624134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:44,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:44,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:44,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708624134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:44,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:44,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708624343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:44,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:44,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708624343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:44,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:44,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:44,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708624344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:44,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:44,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708624344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:44,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,354 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:44,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:44,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708624344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:44,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:44,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:44,410 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/28ef0ee2573042b68181cdf6b7ee1958 2024-12-09T01:42:44,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/31fbf39cc9fc4836a0ad7132e047a247 is 50, key is test_row_0/B:col10/1733708563975/Put/seqid=0 2024-12-09T01:42:44,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742267_1443 (size=12301) 2024-12-09T01:42:44,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708624663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:44,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708624663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:44,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708624664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708624664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:44,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708624664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:44,674 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:44,825 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/31fbf39cc9fc4836a0ad7132e047a247 2024-12-09T01:42:44,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/b4441fbd664548b1b1d48b29565005d4 is 50, key is test_row_0/C:col10/1733708563975/Put/seqid=0 2024-12-09T01:42:44,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742268_1444 (size=12301) 2024-12-09T01:42:45,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:45,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708625184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:45,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:45,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708625184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:45,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:45,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:45,188 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:45,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:45,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:45,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:45,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708625184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:45,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:45,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:45,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:45,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:45,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708625184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:45,188 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:45,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:45,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:45,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:45,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:45,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708625184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:45,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:45,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:45,189 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:45,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/b4441fbd664548b1b1d48b29565005d4 2024-12-09T01:42:45,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/28ef0ee2573042b68181cdf6b7ee1958 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/28ef0ee2573042b68181cdf6b7ee1958 2024-12-09T01:42:45,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/28ef0ee2573042b68181cdf6b7ee1958, entries=200, sequenceid=291, filesize=39.0 K 2024-12-09T01:42:45,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/31fbf39cc9fc4836a0ad7132e047a247 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/31fbf39cc9fc4836a0ad7132e047a247 2024-12-09T01:42:45,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/31fbf39cc9fc4836a0ad7132e047a247, entries=150, sequenceid=291, filesize=12.0 K 2024-12-09T01:42:45,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/b4441fbd664548b1b1d48b29565005d4 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/b4441fbd664548b1b1d48b29565005d4 2024-12-09T01:42:45,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/b4441fbd664548b1b1d48b29565005d4, entries=150, sequenceid=291, filesize=12.0 K 2024-12-09T01:42:45,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 4b9a7812e88070d8f8c92ba7c915f7b9 in 1282ms, sequenceid=291, compaction requested=true 2024-12-09T01:42:45,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:45,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9a7812e88070d8f8c92ba7c915f7b9:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:42:45,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:45,260 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:45,260 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:45,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9a7812e88070d8f8c92ba7c915f7b9:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:42:45,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:45,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9a7812e88070d8f8c92ba7c915f7b9:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:42:45,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:45,261 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:45,261 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102821 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:45,261 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 4b9a7812e88070d8f8c92ba7c915f7b9/B is initiating minor compaction (all files) 2024-12-09T01:42:45,261 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 4b9a7812e88070d8f8c92ba7c915f7b9/A is initiating minor compaction (all files) 2024-12-09T01:42:45,261 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9a7812e88070d8f8c92ba7c915f7b9/B in TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:45,261 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9a7812e88070d8f8c92ba7c915f7b9/A in TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:45,261 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/348f59da2afa48d2ac9f19718f745f45, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/187b947dbdd84a698c61adeb609e2aae, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/28ef0ee2573042b68181cdf6b7ee1958] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp, totalSize=100.4 K 2024-12-09T01:42:45,261 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/b00b50c735504e4ab0c758cb13531e85, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/128e3f7424af48a390926a071fe373d1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/31fbf39cc9fc4836a0ad7132e047a247] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp, totalSize=36.4 K 2024-12-09T01:42:45,261 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:45,261 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. files: [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/348f59da2afa48d2ac9f19718f745f45, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/187b947dbdd84a698c61adeb609e2aae, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/28ef0ee2573042b68181cdf6b7ee1958] 2024-12-09T01:42:45,261 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting b00b50c735504e4ab0c758cb13531e85, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733708560714 2024-12-09T01:42:45,261 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 348f59da2afa48d2ac9f19718f745f45, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733708560714 2024-12-09T01:42:45,262 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 128e3f7424af48a390926a071fe373d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733708561396 2024-12-09T01:42:45,262 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 187b947dbdd84a698c61adeb609e2aae, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733708561396 2024-12-09T01:42:45,262 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 31fbf39cc9fc4836a0ad7132e047a247, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733708563642 2024-12-09T01:42:45,262 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28ef0ee2573042b68181cdf6b7ee1958, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733708563627 2024-12-09T01:42:45,269 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:45,271 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412097007fc70485743fba014b2115ec75210_4b9a7812e88070d8f8c92ba7c915f7b9 store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:45,272 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9a7812e88070d8f8c92ba7c915f7b9#B#compaction#381 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:45,273 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412097007fc70485743fba014b2115ec75210_4b9a7812e88070d8f8c92ba7c915f7b9, store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:45,273 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/6513528a0f0e4395bd601dc05bb9b48b is 50, key is test_row_0/B:col10/1733708563975/Put/seqid=0 2024-12-09T01:42:45,273 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412097007fc70485743fba014b2115ec75210_4b9a7812e88070d8f8c92ba7c915f7b9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:45,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742269_1445 (size=12915) 2024-12-09T01:42:45,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742270_1446 (size=4469) 2024-12-09T01:42:45,286 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9a7812e88070d8f8c92ba7c915f7b9#A#compaction#380 average throughput is 1.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:45,287 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/b890d878fc1740c1836ee832ed2c50c8 is 175, key is test_row_0/A:col10/1733708563975/Put/seqid=0 2024-12-09T01:42:45,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742271_1447 (size=31869) 2024-12-09T01:42:45,297 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/b890d878fc1740c1836ee832ed2c50c8 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/b890d878fc1740c1836ee832ed2c50c8 2024-12-09T01:42:45,306 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b9a7812e88070d8f8c92ba7c915f7b9/A of 4b9a7812e88070d8f8c92ba7c915f7b9 into b890d878fc1740c1836ee832ed2c50c8(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:45,306 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:45,306 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., storeName=4b9a7812e88070d8f8c92ba7c915f7b9/A, priority=13, startTime=1733708565260; duration=0sec 2024-12-09T01:42:45,306 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:45,306 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9a7812e88070d8f8c92ba7c915f7b9:A 2024-12-09T01:42:45,306 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:45,307 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37265 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:45,307 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 4b9a7812e88070d8f8c92ba7c915f7b9/C is initiating minor compaction (all files) 2024-12-09T01:42:45,307 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9a7812e88070d8f8c92ba7c915f7b9/C in TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:45,307 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/d6a74488203b404da1c04836b7ab8e16, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/b6e9c15aae864bf7be295c622196308d, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/b4441fbd664548b1b1d48b29565005d4] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp, totalSize=36.4 K 2024-12-09T01:42:45,307 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6a74488203b404da1c04836b7ab8e16, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1733708560714 2024-12-09T01:42:45,308 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6e9c15aae864bf7be295c622196308d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1733708561396 2024-12-09T01:42:45,308 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4441fbd664548b1b1d48b29565005d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733708563642 2024-12-09T01:42:45,323 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9a7812e88070d8f8c92ba7c915f7b9#C#compaction#382 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:45,324 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/a50a534ab46543809fd8642a2291ca12 is 50, key is test_row_0/C:col10/1733708563975/Put/seqid=0 2024-12-09T01:42:45,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742272_1448 (size=12915) 2024-12-09T01:42:45,338 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/a50a534ab46543809fd8642a2291ca12 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/a50a534ab46543809fd8642a2291ca12 2024-12-09T01:42:45,345 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b9a7812e88070d8f8c92ba7c915f7b9/C of 4b9a7812e88070d8f8c92ba7c915f7b9 into a50a534ab46543809fd8642a2291ca12(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:45,345 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:45,345 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., storeName=4b9a7812e88070d8f8c92ba7c915f7b9/C, priority=13, startTime=1733708565260; duration=0sec 2024-12-09T01:42:45,345 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:45,345 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9a7812e88070d8f8c92ba7c915f7b9:C 2024-12-09T01:42:45,687 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/6513528a0f0e4395bd601dc05bb9b48b as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/6513528a0f0e4395bd601dc05bb9b48b 2024-12-09T01:42:45,692 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b9a7812e88070d8f8c92ba7c915f7b9/B of 4b9a7812e88070d8f8c92ba7c915f7b9 into 6513528a0f0e4395bd601dc05bb9b48b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:45,692 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:45,692 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., storeName=4b9a7812e88070d8f8c92ba7c915f7b9/B, priority=13, startTime=1733708565260; duration=0sec 2024-12-09T01:42:45,692 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:45,692 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9a7812e88070d8f8c92ba7c915f7b9:B 2024-12-09T01:42:45,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-12-09T01:42:45,784 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:45,785 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:42:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-12-09T01:42:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T01:42:45,787 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:42:45,788 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:42:45,788 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:42:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T01:42:45,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-12-09T01:42:45,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:45,940 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 4b9a7812e88070d8f8c92ba7c915f7b9 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T01:42:45,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=A 2024-12-09T01:42:45,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:45,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=B 2024-12-09T01:42:45,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:45,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=C 2024-12-09T01:42:45,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:45,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d00f74c9984a47e88013de13fd78facf_4b9a7812e88070d8f8c92ba7c915f7b9 is 50, key is test_row_0/A:col10/1733708564011/Put/seqid=0 2024-12-09T01:42:45,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742273_1449 (size=12454) 2024-12-09T01:42:45,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:45,958 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209d00f74c9984a47e88013de13fd78facf_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d00f74c9984a47e88013de13fd78facf_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:45,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/51bcc3c2e52746e3b5cadd47aa63c4a5, store: [table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:45,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/51bcc3c2e52746e3b5cadd47aa63c4a5 is 175, key is test_row_0/A:col10/1733708564011/Put/seqid=0 2024-12-09T01:42:45,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742274_1450 (size=31255) 2024-12-09T01:42:45,964 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/51bcc3c2e52746e3b5cadd47aa63c4a5 2024-12-09T01:42:45,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/e2f264e1ba8342d7991638cd6f202ceb is 50, key is test_row_0/B:col10/1733708564011/Put/seqid=0 2024-12-09T01:42:45,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742275_1451 (size=12301) 2024-12-09T01:42:45,991 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/e2f264e1ba8342d7991638cd6f202ceb 2024-12-09T01:42:46,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/06525c87e4da486b8ba46f17c7bc8c6d is 50, key is test_row_0/C:col10/1733708564011/Put/seqid=0 2024-12-09T01:42:46,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742276_1452 (size=12301) 2024-12-09T01:42:46,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T01:42:46,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:46,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:46,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708626212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708626213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708626214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708626218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708626220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708626323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708626323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708626334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,339 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708626334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708626334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,413 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/06525c87e4da486b8ba46f17c7bc8c6d 2024-12-09T01:42:46,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T01:42:46,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/51bcc3c2e52746e3b5cadd47aa63c4a5 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/51bcc3c2e52746e3b5cadd47aa63c4a5 2024-12-09T01:42:46,422 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/51bcc3c2e52746e3b5cadd47aa63c4a5, entries=150, sequenceid=312, filesize=30.5 K 2024-12-09T01:42:46,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/e2f264e1ba8342d7991638cd6f202ceb as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/e2f264e1ba8342d7991638cd6f202ceb 2024-12-09T01:42:46,429 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/e2f264e1ba8342d7991638cd6f202ceb, entries=150, sequenceid=312, filesize=12.0 K 2024-12-09T01:42:46,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/06525c87e4da486b8ba46f17c7bc8c6d as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/06525c87e4da486b8ba46f17c7bc8c6d 2024-12-09T01:42:46,437 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/06525c87e4da486b8ba46f17c7bc8c6d, entries=150, sequenceid=312, filesize=12.0 K 2024-12-09T01:42:46,438 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 4b9a7812e88070d8f8c92ba7c915f7b9 in 498ms, sequenceid=312, compaction requested=false 2024-12-09T01:42:46,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:46,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:46,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-12-09T01:42:46,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-12-09T01:42:46,442 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-12-09T01:42:46,442 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 652 msec 2024-12-09T01:42:46,445 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 658 msec 2024-12-09T01:42:46,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:46,539 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9a7812e88070d8f8c92ba7c915f7b9 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T01:42:46,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=A 2024-12-09T01:42:46,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:46,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=B 2024-12-09T01:42:46,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:46,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=C 2024-12-09T01:42:46,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:46,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b2d2d6923ea84376b486e0fca8fdec05_4b9a7812e88070d8f8c92ba7c915f7b9 is 50, key is test_row_0/A:col10/1733708566218/Put/seqid=0 2024-12-09T01:42:46,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742277_1453 (size=14994) 2024-12-09T01:42:46,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708626568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708626569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708626570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708626579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,595 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708626580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,595 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,595 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708626694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708626694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,703 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708626694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,703 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708626703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708626703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708626913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708626913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708626913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,922 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-09T01:42:46,924 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:46,924 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:42:46,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-12-09T01:42:46,926 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:42:46,927 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:42:46,927 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:42:46,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T01:42:46,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708626924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:46,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708626924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:46,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:46,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:46,963 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:46,967 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b2d2d6923ea84376b486e0fca8fdec05_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b2d2d6923ea84376b486e0fca8fdec05_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:46,968 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/c91bc0387e0a436798634c5ba1394332, store: [table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:46,968 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/c91bc0387e0a436798634c5ba1394332 is 175, key is test_row_0/A:col10/1733708566218/Put/seqid=0 2024-12-09T01:42:46,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742278_1454 (size=39949) 2024-12-09T01:42:47,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T01:42:47,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T01:42:47,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:47,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:47,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:47,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:47,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:47,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:47,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:47,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708627223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:47,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:47,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:47,233 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:47,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T01:42:47,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:47,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:47,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:47,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:47,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:47,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:47,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:47,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708627234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:47,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:47,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708627234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:47,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:47,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T01:42:47,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:47,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708627243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:47,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:47,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:47,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:47,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:47,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708627244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:47,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:47,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:47,257 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:47,373 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=327, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/c91bc0387e0a436798634c5ba1394332 2024-12-09T01:42:47,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/1b1edf3461b644ca84728291253ab40c is 50, key is test_row_0/B:col10/1733708566218/Put/seqid=0 2024-12-09T01:42:47,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T01:42:47,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:47,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:47,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:47,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:47,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:47,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:47,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742279_1455 (size=12301) 2024-12-09T01:42:47,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T01:42:47,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:47,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:47,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:47,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:47,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:47,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:47,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T01:42:47,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T01:42:47,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:47,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:47,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:47,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:47,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:47,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:47,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708627744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:47,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708627753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:47,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708627753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:47,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:47,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:47,766 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:47,766 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:47,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:47,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:47,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:47,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708627763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:47,774 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:47,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:47,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708627764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:47,775 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:47,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/1b1edf3461b644ca84728291253ab40c 2024-12-09T01:42:47,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/5c690a29412349b39df068c0f034277e is 50, key is test_row_0/C:col10/1733708566218/Put/seqid=0 2024-12-09T01:42:47,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742280_1456 (size=12301) 2024-12-09T01:42:47,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T01:42:47,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:47,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:47,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:47,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:47,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:47,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:48,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T01:42:48,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:48,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:48,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:48,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:48,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:48,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T01:42:48,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T01:42:48,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:48,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:48,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:48,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:48,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:48,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:48,214 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=327 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/5c690a29412349b39df068c0f034277e 2024-12-09T01:42:48,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/c91bc0387e0a436798634c5ba1394332 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/c91bc0387e0a436798634c5ba1394332 2024-12-09T01:42:48,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/c91bc0387e0a436798634c5ba1394332, entries=200, sequenceid=327, filesize=39.0 K 2024-12-09T01:42:48,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/1b1edf3461b644ca84728291253ab40c as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/1b1edf3461b644ca84728291253ab40c 2024-12-09T01:42:48,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/1b1edf3461b644ca84728291253ab40c, entries=150, sequenceid=327, filesize=12.0 K 2024-12-09T01:42:48,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/5c690a29412349b39df068c0f034277e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/5c690a29412349b39df068c0f034277e 2024-12-09T01:42:48,229 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/5c690a29412349b39df068c0f034277e, entries=150, sequenceid=327, filesize=12.0 K 2024-12-09T01:42:48,230 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 4b9a7812e88070d8f8c92ba7c915f7b9 in 1692ms, sequenceid=327, compaction requested=true 2024-12-09T01:42:48,230 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:48,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9a7812e88070d8f8c92ba7c915f7b9:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:42:48,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:48,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9a7812e88070d8f8c92ba7c915f7b9:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:42:48,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:48,230 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:48,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9a7812e88070d8f8c92ba7c915f7b9:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:42:48,230 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:48,230 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:48,231 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:48,231 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 4b9a7812e88070d8f8c92ba7c915f7b9/B is initiating minor compaction (all files) 2024-12-09T01:42:48,231 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9a7812e88070d8f8c92ba7c915f7b9/B in TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:48,231 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/6513528a0f0e4395bd601dc05bb9b48b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/e2f264e1ba8342d7991638cd6f202ceb, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/1b1edf3461b644ca84728291253ab40c] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp, totalSize=36.6 K 2024-12-09T01:42:48,232 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 6513528a0f0e4395bd601dc05bb9b48b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733708563642 2024-12-09T01:42:48,232 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:48,232 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 4b9a7812e88070d8f8c92ba7c915f7b9/A is initiating minor compaction (all files) 2024-12-09T01:42:48,232 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting e2f264e1ba8342d7991638cd6f202ceb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733708564010 2024-12-09T01:42:48,232 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9a7812e88070d8f8c92ba7c915f7b9/A in TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:48,232 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/b890d878fc1740c1836ee832ed2c50c8, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/51bcc3c2e52746e3b5cadd47aa63c4a5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/c91bc0387e0a436798634c5ba1394332] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp, totalSize=100.7 K 2024-12-09T01:42:48,232 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:48,233 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. files: [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/b890d878fc1740c1836ee832ed2c50c8, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/51bcc3c2e52746e3b5cadd47aa63c4a5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/c91bc0387e0a436798634c5ba1394332] 2024-12-09T01:42:48,233 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b1edf3461b644ca84728291253ab40c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733708566217 2024-12-09T01:42:48,233 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting b890d878fc1740c1836ee832ed2c50c8, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733708563642 2024-12-09T01:42:48,233 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51bcc3c2e52746e3b5cadd47aa63c4a5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733708564010 2024-12-09T01:42:48,234 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting c91bc0387e0a436798634c5ba1394332, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733708566209 2024-12-09T01:42:48,243 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9a7812e88070d8f8c92ba7c915f7b9#B#compaction#389 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:48,243 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/877af8ac1213499daa1244f61f183b6f is 50, key is test_row_0/B:col10/1733708566218/Put/seqid=0 2024-12-09T01:42:48,244 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:48,247 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120994e18c9635944f6e8388e19773ff588e_4b9a7812e88070d8f8c92ba7c915f7b9 store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:48,248 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120994e18c9635944f6e8388e19773ff588e_4b9a7812e88070d8f8c92ba7c915f7b9, store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:48,248 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120994e18c9635944f6e8388e19773ff588e_4b9a7812e88070d8f8c92ba7c915f7b9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:48,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742281_1457 (size=13017) 2024-12-09T01:42:48,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742282_1458 (size=4469) 2024-12-09T01:42:48,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-12-09T01:42:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:48,307 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 4b9a7812e88070d8f8c92ba7c915f7b9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-09T01:42:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=A 2024-12-09T01:42:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=B 2024-12-09T01:42:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=C 2024-12-09T01:42:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:48,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092211944569794335918489b47a1bdec0_4b9a7812e88070d8f8c92ba7c915f7b9 is 50, key is test_row_0/A:col10/1733708566569/Put/seqid=0 2024-12-09T01:42:48,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742283_1459 (size=12454) 2024-12-09T01:42:48,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:48,322 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092211944569794335918489b47a1bdec0_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092211944569794335918489b47a1bdec0_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:48,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/ccc888db1d744d84aedd1a23a5a390da, store: [table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:48,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/ccc888db1d744d84aedd1a23a5a390da is 175, key is test_row_0/A:col10/1733708566569/Put/seqid=0 2024-12-09T01:42:48,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742284_1460 (size=31255) 2024-12-09T01:42:48,656 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/877af8ac1213499daa1244f61f183b6f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/877af8ac1213499daa1244f61f183b6f 2024-12-09T01:42:48,658 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9a7812e88070d8f8c92ba7c915f7b9#A#compaction#390 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:48,659 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/7e87d3cf6f9a46059d3964bee73fda04 is 175, key is test_row_0/A:col10/1733708566218/Put/seqid=0 2024-12-09T01:42:48,663 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b9a7812e88070d8f8c92ba7c915f7b9/B of 4b9a7812e88070d8f8c92ba7c915f7b9 into 877af8ac1213499daa1244f61f183b6f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:48,663 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:48,663 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., storeName=4b9a7812e88070d8f8c92ba7c915f7b9/B, priority=13, startTime=1733708568230; duration=0sec 2024-12-09T01:42:48,663 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:48,663 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9a7812e88070d8f8c92ba7c915f7b9:B 2024-12-09T01:42:48,664 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:48,667 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:48,667 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 4b9a7812e88070d8f8c92ba7c915f7b9/C is initiating minor compaction (all files) 2024-12-09T01:42:48,667 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9a7812e88070d8f8c92ba7c915f7b9/C in TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:48,667 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/a50a534ab46543809fd8642a2291ca12, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/06525c87e4da486b8ba46f17c7bc8c6d, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/5c690a29412349b39df068c0f034277e] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp, totalSize=36.6 K 2024-12-09T01:42:48,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742285_1461 (size=31971) 2024-12-09T01:42:48,668 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting a50a534ab46543809fd8642a2291ca12, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1733708563642 2024-12-09T01:42:48,668 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 06525c87e4da486b8ba46f17c7bc8c6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1733708564010 2024-12-09T01:42:48,669 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c690a29412349b39df068c0f034277e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733708566217 2024-12-09T01:42:48,678 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9a7812e88070d8f8c92ba7c915f7b9#C#compaction#392 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:48,679 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/21c9c935ada1430ba52a8fb51451cac4 is 50, key is test_row_0/C:col10/1733708566218/Put/seqid=0 2024-12-09T01:42:48,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742286_1462 (size=13017) 2024-12-09T01:42:48,692 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/21c9c935ada1430ba52a8fb51451cac4 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/21c9c935ada1430ba52a8fb51451cac4 2024-12-09T01:42:48,701 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b9a7812e88070d8f8c92ba7c915f7b9/C of 4b9a7812e88070d8f8c92ba7c915f7b9 into 21c9c935ada1430ba52a8fb51451cac4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:48,701 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:48,701 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., storeName=4b9a7812e88070d8f8c92ba7c915f7b9/C, priority=13, startTime=1733708568230; duration=0sec 2024-12-09T01:42:48,701 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:48,701 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9a7812e88070d8f8c92ba7c915f7b9:C 2024-12-09T01:42:48,735 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=344, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/ccc888db1d744d84aedd1a23a5a390da 2024-12-09T01:42:48,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/bdfa8a3284d24510a007c4c117a4d870 is 50, key is test_row_0/B:col10/1733708566569/Put/seqid=0 2024-12-09T01:42:48,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742287_1463 (size=12301) 2024-12-09T01:42:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:48,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. as already flushing 2024-12-09T01:42:48,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:48,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708628800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:48,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:48,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:48,815 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:48,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:48,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708628811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:48,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:48,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:48,816 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:48,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:48,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708628812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:48,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:48,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:48,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708628814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:48,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:48,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:48,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:48,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:48,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:48,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708628816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:48,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:48,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:48,832 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:48,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708628924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:48,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708628924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:48,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708628933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:48,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:48,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:48,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:48,941 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:48,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708628933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:48,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:48,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:48,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:48,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:48,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708628943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:48,955 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:48,956 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:49,073 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/7e87d3cf6f9a46059d3964bee73fda04 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/7e87d3cf6f9a46059d3964bee73fda04 2024-12-09T01:42:49,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T01:42:49,078 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b9a7812e88070d8f8c92ba7c915f7b9/A of 4b9a7812e88070d8f8c92ba7c915f7b9 into 7e87d3cf6f9a46059d3964bee73fda04(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:49,078 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:49,078 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., storeName=4b9a7812e88070d8f8c92ba7c915f7b9/A, priority=13, startTime=1733708568230; duration=0sec 2024-12-09T01:42:49,078 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:49,078 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9a7812e88070d8f8c92ba7c915f7b9:A 2024-12-09T01:42:49,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:49,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708629133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:49,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:49,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708629133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:49,154 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/bdfa8a3284d24510a007c4c117a4d870 2024-12-09T01:42:49,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:49,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708629143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:49,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:49,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708629143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:49,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,155 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:49,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:49,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/d80510125ad042d0b975486bf2c1cd45 is 50, key is test_row_0/C:col10/1733708566569/Put/seqid=0 2024-12-09T01:42:49,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742288_1464 (size=12301) 2024-12-09T01:42:49,166 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/d80510125ad042d0b975486bf2c1cd45 2024-12-09T01:42:49,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/ccc888db1d744d84aedd1a23a5a390da as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/ccc888db1d744d84aedd1a23a5a390da 2024-12-09T01:42:49,174 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/ccc888db1d744d84aedd1a23a5a390da, entries=150, sequenceid=344, filesize=30.5 K 2024-12-09T01:42:49,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/bdfa8a3284d24510a007c4c117a4d870 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/bdfa8a3284d24510a007c4c117a4d870 2024-12-09T01:42:49,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:49,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708629164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:49,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:49,179 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/bdfa8a3284d24510a007c4c117a4d870, entries=150, sequenceid=344, filesize=12.0 K 2024-12-09T01:42:49,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/d80510125ad042d0b975486bf2c1cd45 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/d80510125ad042d0b975486bf2c1cd45 2024-12-09T01:42:49,183 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/d80510125ad042d0b975486bf2c1cd45, entries=150, sequenceid=344, filesize=12.0 K 2024-12-09T01:42:49,184 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 4b9a7812e88070d8f8c92ba7c915f7b9 in 877ms, sequenceid=344, compaction requested=false 2024-12-09T01:42:49,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:49,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:49,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-12-09T01:42:49,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-12-09T01:42:49,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-12-09T01:42:49,188 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2590 sec 2024-12-09T01:42:49,191 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.2650 sec 2024-12-09T01:42:49,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:49,447 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9a7812e88070d8f8c92ba7c915f7b9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-09T01:42:49,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=A 2024-12-09T01:42:49,447 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:49,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=B 2024-12-09T01:42:49,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:49,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=C 2024-12-09T01:42:49,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:49,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c8cd686929b84c9bb3a38455c9ea0b08_4b9a7812e88070d8f8c92ba7c915f7b9 is 50, key is test_row_0/A:col10/1733708568812/Put/seqid=0 2024-12-09T01:42:49,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742289_1465 (size=14994) 2024-12-09T01:42:49,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708629484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708629485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:49,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708629495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:49,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:49,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708629498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:49,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:49,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708629498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:49,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,513 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:49,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708629603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:49,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708629604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:49,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708629623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:49,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708629623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:49,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,627 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:49,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:49,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708629813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:49,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:49,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708629814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:49,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:49,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:49,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:49,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708629833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:49,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708629833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:49,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:49,846 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:49,864 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:49,867 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c8cd686929b84c9bb3a38455c9ea0b08_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c8cd686929b84c9bb3a38455c9ea0b08_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:49,868 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/3354af69178145ec9701244cae2f59b0, store: [table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:49,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/3354af69178145ec9701244cae2f59b0 is 175, key is test_row_0/A:col10/1733708568812/Put/seqid=0 2024-12-09T01:42:49,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742290_1466 (size=39949) 2024-12-09T01:42:50,005 INFO [Thread-1688 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1688. 2024-12-09T01:42:50,005 DEBUG [Thread-1688 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:42:50,005 DEBUG [Thread-1688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:50,005 DEBUG [Thread-1688 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:50,005 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:42:50,009 INFO [Thread-1696 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1696. 2024-12-09T01:42:50,009 DEBUG [Thread-1696 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:42:50,009 DEBUG [Thread-1696 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:50,009 DEBUG [Thread-1696 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:50,009 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:42:50,010 INFO [Thread-1690 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1690. 2024-12-09T01:42:50,010 DEBUG [Thread-1690 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:42:50,010 DEBUG [Thread-1690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:50,010 DEBUG [Thread-1690 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:50,010 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:42:50,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708630013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:50,014 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:50,017 INFO [Thread-1694 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1694. 2024-12-09T01:42:50,017 DEBUG [Thread-1694 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:42:50,017 DEBUG [Thread-1694 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:50,017 DEBUG [Thread-1694 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:50,017 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:42:50,020 INFO [Thread-1692 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1692. 2024-12-09T01:42:50,020 DEBUG [Thread-1692 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:42:50,020 DEBUG [Thread-1692 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:50,020 DEBUG [Thread-1692 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:50,020 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:42:50,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:50,134 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708630134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:50,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708630134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:50,134 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:50,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:50,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:50,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708630153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:50,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708630153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:50,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:50,274 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/3354af69178145ec9701244cae2f59b0 2024-12-09T01:42:50,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/dcf3c318c58245d79cf003d9dee70cdc is 50, key is test_row_0/B:col10/1733708568812/Put/seqid=0 2024-12-09T01:42:50,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742291_1467 (size=12301) 2024-12-09T01:42:50,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:50,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:50,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47958 deadline: 1733708630644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:50,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47968 deadline: 1733708630644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:50,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:50,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:50,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:50,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:50,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:50,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:50,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:50,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:50,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47966 deadline: 1733708630666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:50,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47944 deadline: 1733708630666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:50,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:50,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:50,667 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:50,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:50,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:50,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:50,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/dcf3c318c58245d79cf003d9dee70cdc 2024-12-09T01:42:50,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/f5821fdd5bbb4792a63fc45f7670b550 is 50, key is test_row_0/C:col10/1733708568812/Put/seqid=0 2024-12-09T01:42:50,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742292_1468 (size=12301) 2024-12-09T01:42:51,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:51,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48034 deadline: 1733708631024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:51,025 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_2' in 4b9a7812e88070d8f8c92ba7c915f7b9 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:42:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:51,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:42:51,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-09T01:42:51,084 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:51,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/f5821fdd5bbb4792a63fc45f7670b550 2024-12-09T01:42:51,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/3354af69178145ec9701244cae2f59b0 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3354af69178145ec9701244cae2f59b0 2024-12-09T01:42:51,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3354af69178145ec9701244cae2f59b0, entries=200, sequenceid=364, filesize=39.0 K 2024-12-09T01:42:51,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/dcf3c318c58245d79cf003d9dee70cdc as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/dcf3c318c58245d79cf003d9dee70cdc 2024-12-09T01:42:51,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/dcf3c318c58245d79cf003d9dee70cdc, entries=150, sequenceid=364, filesize=12.0 K 2024-12-09T01:42:51,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/f5821fdd5bbb4792a63fc45f7670b550 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/f5821fdd5bbb4792a63fc45f7670b550 2024-12-09T01:42:51,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/f5821fdd5bbb4792a63fc45f7670b550, entries=150, sequenceid=364, filesize=12.0 K 2024-12-09T01:42:51,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 4b9a7812e88070d8f8c92ba7c915f7b9 in 1667ms, sequenceid=364, compaction requested=true 2024-12-09T01:42:51,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:51,114 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:51,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9a7812e88070d8f8c92ba7c915f7b9:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:42:51,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:51,115 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:51,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9a7812e88070d8f8c92ba7c915f7b9:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:42:51,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:51,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4b9a7812e88070d8f8c92ba7c915f7b9:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:42:51,115 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:51,115 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:51,115 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 4b9a7812e88070d8f8c92ba7c915f7b9/A is initiating minor compaction (all files) 2024-12-09T01:42:51,115 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9a7812e88070d8f8c92ba7c915f7b9/A in TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:51,115 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/7e87d3cf6f9a46059d3964bee73fda04, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/ccc888db1d744d84aedd1a23a5a390da, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3354af69178145ec9701244cae2f59b0] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp, totalSize=100.8 K 2024-12-09T01:42:51,116 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:51,116 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. files: [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/7e87d3cf6f9a46059d3964bee73fda04, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/ccc888db1d744d84aedd1a23a5a390da, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3354af69178145ec9701244cae2f59b0] 2024-12-09T01:42:51,116 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e87d3cf6f9a46059d3964bee73fda04, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733708566217 2024-12-09T01:42:51,116 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:51,116 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 4b9a7812e88070d8f8c92ba7c915f7b9/B is initiating minor compaction (all files) 2024-12-09T01:42:51,116 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9a7812e88070d8f8c92ba7c915f7b9/B in TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:51,117 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/877af8ac1213499daa1244f61f183b6f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/bdfa8a3284d24510a007c4c117a4d870, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/dcf3c318c58245d79cf003d9dee70cdc] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp, totalSize=36.7 K 2024-12-09T01:42:51,117 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccc888db1d744d84aedd1a23a5a390da, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733708566569 2024-12-09T01:42:51,117 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 877af8ac1213499daa1244f61f183b6f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733708566217 2024-12-09T01:42:51,117 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3354af69178145ec9701244cae2f59b0, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733708568798 2024-12-09T01:42:51,117 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting bdfa8a3284d24510a007c4c117a4d870, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733708566569 2024-12-09T01:42:51,118 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting dcf3c318c58245d79cf003d9dee70cdc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733708568798 2024-12-09T01:42:51,132 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:51,135 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9a7812e88070d8f8c92ba7c915f7b9#B#compaction#398 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:51,136 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/3359ff8457984d59b66863d3b4d6a942 is 50, key is test_row_0/B:col10/1733708568812/Put/seqid=0 2024-12-09T01:42:51,144 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120915fdb8195d0d4454b98e0acff3215d8b_4b9a7812e88070d8f8c92ba7c915f7b9 store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:51,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742293_1469 (size=13119) 2024-12-09T01:42:51,160 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/3359ff8457984d59b66863d3b4d6a942 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/3359ff8457984d59b66863d3b4d6a942 2024-12-09T01:42:51,165 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b9a7812e88070d8f8c92ba7c915f7b9/B of 4b9a7812e88070d8f8c92ba7c915f7b9 into 3359ff8457984d59b66863d3b4d6a942(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:51,165 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:51,165 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., storeName=4b9a7812e88070d8f8c92ba7c915f7b9/B, priority=13, startTime=1733708571114; duration=0sec 2024-12-09T01:42:51,165 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:51,165 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9a7812e88070d8f8c92ba7c915f7b9:B 2024-12-09T01:42:51,165 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:51,166 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:51,166 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 4b9a7812e88070d8f8c92ba7c915f7b9/C is initiating minor compaction (all files) 2024-12-09T01:42:51,166 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4b9a7812e88070d8f8c92ba7c915f7b9/C in TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:51,166 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/21c9c935ada1430ba52a8fb51451cac4, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/d80510125ad042d0b975486bf2c1cd45, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/f5821fdd5bbb4792a63fc45f7670b550] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp, totalSize=36.7 K 2024-12-09T01:42:51,167 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 21c9c935ada1430ba52a8fb51451cac4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=327, earliestPutTs=1733708566217 2024-12-09T01:42:51,167 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting d80510125ad042d0b975486bf2c1cd45, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1733708566569 2024-12-09T01:42:51,167 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting f5821fdd5bbb4792a63fc45f7670b550, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1733708568798 2024-12-09T01:42:51,184 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9a7812e88070d8f8c92ba7c915f7b9#C#compaction#400 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:51,185 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/8ef82a7288c0489f9d7a537a27e0a1cd is 50, key is test_row_0/C:col10/1733708568812/Put/seqid=0 2024-12-09T01:42:51,201 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120915fdb8195d0d4454b98e0acff3215d8b_4b9a7812e88070d8f8c92ba7c915f7b9, store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:51,201 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120915fdb8195d0d4454b98e0acff3215d8b_4b9a7812e88070d8f8c92ba7c915f7b9 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:51,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742295_1471 (size=4469) 2024-12-09T01:42:51,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742294_1470 (size=13119) 2024-12-09T01:42:51,654 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4b9a7812e88070d8f8c92ba7c915f7b9#A#compaction#399 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:51,655 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/ce86752f9ed9444592528bbff7c0387f is 175, key is test_row_0/A:col10/1733708568812/Put/seqid=0 2024-12-09T01:42:51,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:51,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4b9a7812e88070d8f8c92ba7c915f7b9 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-09T01:42:51,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=A 2024-12-09T01:42:51,656 INFO [Thread-1681 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1681. 2024-12-09T01:42:51,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:51,656 DEBUG [Thread-1681 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:42:51,656 DEBUG [Thread-1681 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:51,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=B 2024-12-09T01:42:51,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:51,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=C 2024-12-09T01:42:51,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:51,656 DEBUG [Thread-1681 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:51,656 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:42:51,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742296_1472 (size=32073) 2024-12-09T01:42:51,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412094b97cf7ac8434c309817161793cabe25_4b9a7812e88070d8f8c92ba7c915f7b9 is 50, key is test_row_0/A:col10/1733708569495/Put/seqid=0 2024-12-09T01:42:51,665 INFO [Thread-1679 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1679. 2024-12-09T01:42:51,665 DEBUG [Thread-1679 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:42:51,665 DEBUG [Thread-1679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:51,665 DEBUG [Thread-1679 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:51,665 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:42:51,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742297_1473 (size=12454) 2024-12-09T01:42:51,673 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/8ef82a7288c0489f9d7a537a27e0a1cd as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/8ef82a7288c0489f9d7a537a27e0a1cd 2024-12-09T01:42:51,675 INFO [Thread-1685 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1685. 2024-12-09T01:42:51,675 DEBUG [Thread-1685 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:42:51,675 DEBUG [Thread-1685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:51,675 DEBUG [Thread-1685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:51,675 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:42:51,677 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b9a7812e88070d8f8c92ba7c915f7b9/C of 4b9a7812e88070d8f8c92ba7c915f7b9 into 8ef82a7288c0489f9d7a537a27e0a1cd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:51,677 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:51,677 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., storeName=4b9a7812e88070d8f8c92ba7c915f7b9/C, priority=13, startTime=1733708571115; duration=0sec 2024-12-09T01:42:51,677 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:51,677 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9a7812e88070d8f8c92ba7c915f7b9:C 2024-12-09T01:42:51,685 INFO [Thread-1677 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1677. 2024-12-09T01:42:51,685 DEBUG [Thread-1677 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:42:51,685 DEBUG [Thread-1677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:51,685 DEBUG [Thread-1677 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:51,685 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:42:51,959 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T01:42:52,063 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/ce86752f9ed9444592528bbff7c0387f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/ce86752f9ed9444592528bbff7c0387f 2024-12-09T01:42:52,067 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4b9a7812e88070d8f8c92ba7c915f7b9/A of 4b9a7812e88070d8f8c92ba7c915f7b9 into ce86752f9ed9444592528bbff7c0387f(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:52,067 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:52,067 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., storeName=4b9a7812e88070d8f8c92ba7c915f7b9/A, priority=13, startTime=1733708571114; duration=0sec 2024-12-09T01:42:52,068 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:52,068 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4b9a7812e88070d8f8c92ba7c915f7b9:A 2024-12-09T01:42:52,068 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:52,071 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412094b97cf7ac8434c309817161793cabe25_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094b97cf7ac8434c309817161793cabe25_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:52,078 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/e7a433d594b64eea9d0d947d84ab8312, store: [table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:52,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/e7a433d594b64eea9d0d947d84ab8312 is 175, key is test_row_0/A:col10/1733708569495/Put/seqid=0 2024-12-09T01:42:52,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742298_1474 (size=31255) 2024-12-09T01:42:52,087 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=382, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/e7a433d594b64eea9d0d947d84ab8312 2024-12-09T01:42:52,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/d560d64361f54e3480d4521f4cdff21b is 50, key is test_row_0/B:col10/1733708569495/Put/seqid=0 2024-12-09T01:42:52,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742299_1475 (size=12301) 2024-12-09T01:42:52,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/d560d64361f54e3480d4521f4cdff21b 2024-12-09T01:42:52,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/ab7072efe6664ad1b47e53f41c2c8d4c is 50, key is test_row_0/C:col10/1733708569495/Put/seqid=0 2024-12-09T01:42:52,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742300_1476 (size=12301) 2024-12-09T01:42:52,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=382 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/ab7072efe6664ad1b47e53f41c2c8d4c 2024-12-09T01:42:52,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/e7a433d594b64eea9d0d947d84ab8312 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/e7a433d594b64eea9d0d947d84ab8312 2024-12-09T01:42:52,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/e7a433d594b64eea9d0d947d84ab8312, entries=150, sequenceid=382, filesize=30.5 K 2024-12-09T01:42:52,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/d560d64361f54e3480d4521f4cdff21b as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/d560d64361f54e3480d4521f4cdff21b 2024-12-09T01:42:52,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/d560d64361f54e3480d4521f4cdff21b, entries=150, sequenceid=382, filesize=12.0 K 2024-12-09T01:42:52,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/ab7072efe6664ad1b47e53f41c2c8d4c as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/ab7072efe6664ad1b47e53f41c2c8d4c 2024-12-09T01:42:52,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/ab7072efe6664ad1b47e53f41c2c8d4c, entries=150, sequenceid=382, filesize=12.0 K 2024-12-09T01:42:52,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=20.13 KB/20610 for 4b9a7812e88070d8f8c92ba7c915f7b9 in 889ms, sequenceid=382, compaction requested=false 2024-12-09T01:42:52,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4b9a7812e88070d8f8c92ba7c915f7b9: 2024-12-09T01:42:53,056 INFO [Thread-1683 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1683. 2024-12-09T01:42:53,056 DEBUG [Thread-1683 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:42:53,056 DEBUG [Thread-1683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:53,056 DEBUG [Thread-1683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:53,056 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:42:53,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T01:42:53,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-12-09T01:42:53,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-12-09T01:42:53,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-12-09T01:42:53,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-12-09T01:42:53,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-12-09T01:42:53,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T01:42:53,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T01:42:53,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1075 2024-12-09T01:42:53,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3224 rows 2024-12-09T01:42:53,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1075 2024-12-09T01:42:53,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3224 rows 2024-12-09T01:42:53,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1078 2024-12-09T01:42:53,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3233 rows 2024-12-09T01:42:53,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1078 2024-12-09T01:42:53,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3234 rows 2024-12-09T01:42:53,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1077 2024-12-09T01:42:53,057 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3230 rows 2024-12-09T01:42:53,057 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T01:42:53,057 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T01:42:53,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:53,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:53,058 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:42:53,058 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T01:42:53,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:53,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T01:42:53,061 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708573061"}]},"ts":"1733708573061"} 2024-12-09T01:42:53,063 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T01:42:53,063 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T01:42:53,063 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T01:42:53,065 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b9a7812e88070d8f8c92ba7c915f7b9, UNASSIGN}] 2024-12-09T01:42:53,066 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b9a7812e88070d8f8c92ba7c915f7b9, UNASSIGN 2024-12-09T01:42:53,068 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=4b9a7812e88070d8f8c92ba7c915f7b9, regionState=CLOSING, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:53,069 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b9a7812e88070d8f8c92ba7c915f7b9, UNASSIGN because future has completed 2024-12-09T01:42:53,070 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T01:42:53,070 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943}] 2024-12-09T01:42:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T01:42:53,222 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:53,222 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T01:42:53,222 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing 4b9a7812e88070d8f8c92ba7c915f7b9, disabling compactions & flushes 2024-12-09T01:42:53,222 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:53,222 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:53,222 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. after waiting 0 ms 2024-12-09T01:42:53,222 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:53,222 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing 4b9a7812e88070d8f8c92ba7c915f7b9 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-09T01:42:53,222 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=A 2024-12-09T01:42:53,223 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:53,223 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=B 2024-12-09T01:42:53,223 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:53,223 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 4b9a7812e88070d8f8c92ba7c915f7b9, store=C 2024-12-09T01:42:53,223 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:53,228 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120954be52780a7148208ff460073793593a_4b9a7812e88070d8f8c92ba7c915f7b9 is 50, key is test_row_0/A:col10/1733708571684/Put/seqid=0 2024-12-09T01:42:53,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742301_1477 (size=12454) 2024-12-09T01:42:53,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T01:42:53,631 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:53,634 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120954be52780a7148208ff460073793593a_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120954be52780a7148208ff460073793593a_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:53,635 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/d0080082c42e4f6aa1f5a929f25a0b19, store: [table=TestAcidGuarantees family=A region=4b9a7812e88070d8f8c92ba7c915f7b9] 2024-12-09T01:42:53,636 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/d0080082c42e4f6aa1f5a929f25a0b19 is 175, key is test_row_0/A:col10/1733708571684/Put/seqid=0 2024-12-09T01:42:53,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742302_1478 (size=31255) 2024-12-09T01:42:53,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T01:42:54,039 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=391, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/d0080082c42e4f6aa1f5a929f25a0b19 2024-12-09T01:42:54,045 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/ef43493edea84c6b8c7d46e12fdf963b is 50, key is test_row_0/B:col10/1733708571684/Put/seqid=0 2024-12-09T01:42:54,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742303_1479 (size=12301) 2024-12-09T01:42:54,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T01:42:54,448 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/ef43493edea84c6b8c7d46e12fdf963b 2024-12-09T01:42:54,454 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/f08590770292469cb5c09ab4ac4c55eb is 50, key is test_row_0/C:col10/1733708571684/Put/seqid=0 2024-12-09T01:42:54,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742304_1480 (size=12301) 2024-12-09T01:42:54,858 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/f08590770292469cb5c09ab4ac4c55eb 2024-12-09T01:42:54,861 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/A/d0080082c42e4f6aa1f5a929f25a0b19 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/d0080082c42e4f6aa1f5a929f25a0b19 2024-12-09T01:42:54,865 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/d0080082c42e4f6aa1f5a929f25a0b19, entries=150, sequenceid=391, filesize=30.5 K 2024-12-09T01:42:54,865 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/B/ef43493edea84c6b8c7d46e12fdf963b as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/ef43493edea84c6b8c7d46e12fdf963b 2024-12-09T01:42:54,868 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/ef43493edea84c6b8c7d46e12fdf963b, entries=150, sequenceid=391, filesize=12.0 K 2024-12-09T01:42:54,869 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/.tmp/C/f08590770292469cb5c09ab4ac4c55eb as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/f08590770292469cb5c09ab4ac4c55eb 2024-12-09T01:42:54,872 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/f08590770292469cb5c09ab4ac4c55eb, entries=150, sequenceid=391, filesize=12.0 K 2024-12-09T01:42:54,873 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 4b9a7812e88070d8f8c92ba7c915f7b9 in 1651ms, sequenceid=391, compaction requested=true 2024-12-09T01:42:54,873 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3eaa7d2380414f249a73ca57ac671d69, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/18ffadd011774f129bfd8ee6b55d27c9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3cac59c4c1bf4018860b3a86eb5d1ccf, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/8a07a28fa0f74cada5b7908a54a0a3df, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/e3645cd605704de8b58529fcfe8b2a8e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/6e64cc14fbdc4c4fb51905273680a2ee, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/622fd52003d340f7ab570f853ab659ba, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/5410b6ac536b4430906c30a0473290da, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/20695f4409b6469e9d8186595055ecc8, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3eb534ed50cb451590ecfc9f7b6a0abc, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/07222b09e39448d68010faa1d88d8ac3, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/16d3e1ccc55646a69698fe5f7e844ca0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/52e51917b5a2412eb17e7259ebaaeabb, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/68e683e0f4fc47eea62b3d1c61820343, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/00f1a68da3ef4f94a54bd6d50c346450, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/348f59da2afa48d2ac9f19718f745f45, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/187b947dbdd84a698c61adeb609e2aae, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/28ef0ee2573042b68181cdf6b7ee1958, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/b890d878fc1740c1836ee832ed2c50c8, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/51bcc3c2e52746e3b5cadd47aa63c4a5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/c91bc0387e0a436798634c5ba1394332, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/7e87d3cf6f9a46059d3964bee73fda04, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/ccc888db1d744d84aedd1a23a5a390da, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3354af69178145ec9701244cae2f59b0] to archive 2024-12-09T01:42:54,874 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T01:42:54,875 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3eaa7d2380414f249a73ca57ac671d69 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3eaa7d2380414f249a73ca57ac671d69 2024-12-09T01:42:54,876 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/18ffadd011774f129bfd8ee6b55d27c9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/18ffadd011774f129bfd8ee6b55d27c9 2024-12-09T01:42:54,877 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3cac59c4c1bf4018860b3a86eb5d1ccf to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3cac59c4c1bf4018860b3a86eb5d1ccf 2024-12-09T01:42:54,878 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/8a07a28fa0f74cada5b7908a54a0a3df to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/8a07a28fa0f74cada5b7908a54a0a3df 2024-12-09T01:42:54,879 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/e3645cd605704de8b58529fcfe8b2a8e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/e3645cd605704de8b58529fcfe8b2a8e 2024-12-09T01:42:54,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/6e64cc14fbdc4c4fb51905273680a2ee to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/6e64cc14fbdc4c4fb51905273680a2ee 2024-12-09T01:42:54,880 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/622fd52003d340f7ab570f853ab659ba to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/622fd52003d340f7ab570f853ab659ba 2024-12-09T01:42:54,881 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/5410b6ac536b4430906c30a0473290da to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/5410b6ac536b4430906c30a0473290da 2024-12-09T01:42:54,882 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/20695f4409b6469e9d8186595055ecc8 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/20695f4409b6469e9d8186595055ecc8 2024-12-09T01:42:54,883 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3eb534ed50cb451590ecfc9f7b6a0abc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3eb534ed50cb451590ecfc9f7b6a0abc 2024-12-09T01:42:54,884 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/07222b09e39448d68010faa1d88d8ac3 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/07222b09e39448d68010faa1d88d8ac3 2024-12-09T01:42:54,885 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/16d3e1ccc55646a69698fe5f7e844ca0 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/16d3e1ccc55646a69698fe5f7e844ca0 2024-12-09T01:42:54,886 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/52e51917b5a2412eb17e7259ebaaeabb to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/52e51917b5a2412eb17e7259ebaaeabb 2024-12-09T01:42:54,887 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/68e683e0f4fc47eea62b3d1c61820343 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/68e683e0f4fc47eea62b3d1c61820343 2024-12-09T01:42:54,888 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/00f1a68da3ef4f94a54bd6d50c346450 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/00f1a68da3ef4f94a54bd6d50c346450 2024-12-09T01:42:54,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/348f59da2afa48d2ac9f19718f745f45 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/348f59da2afa48d2ac9f19718f745f45 2024-12-09T01:42:54,889 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/187b947dbdd84a698c61adeb609e2aae to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/187b947dbdd84a698c61adeb609e2aae 2024-12-09T01:42:54,890 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/28ef0ee2573042b68181cdf6b7ee1958 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/28ef0ee2573042b68181cdf6b7ee1958 2024-12-09T01:42:54,891 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/b890d878fc1740c1836ee832ed2c50c8 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/b890d878fc1740c1836ee832ed2c50c8 2024-12-09T01:42:54,892 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/51bcc3c2e52746e3b5cadd47aa63c4a5 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/51bcc3c2e52746e3b5cadd47aa63c4a5 2024-12-09T01:42:54,893 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/c91bc0387e0a436798634c5ba1394332 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/c91bc0387e0a436798634c5ba1394332 2024-12-09T01:42:54,894 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/7e87d3cf6f9a46059d3964bee73fda04 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/7e87d3cf6f9a46059d3964bee73fda04 2024-12-09T01:42:54,895 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/ccc888db1d744d84aedd1a23a5a390da to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/ccc888db1d744d84aedd1a23a5a390da 2024-12-09T01:42:54,896 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3354af69178145ec9701244cae2f59b0 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/3354af69178145ec9701244cae2f59b0 2024-12-09T01:42:54,897 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/2b319aceac1d472891fd0329c33735b6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/a32ab115f9e449968f9e1d2e45fe4034, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/b5b11f7000d5436ea9432668c4100c13, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/e92ad5d8c4d045c889ffaa9e565c6107, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/5960d5885b2d4974b7d11b2f8657a409, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/b7494de9f7444c8d83457c32f4245380, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/6c9ceb64995049628f721870cefc7db0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/3ef536540d8a4c88a93816e0ac58d095, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/bfb214fea9874901a6bec879a9a6ff36, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/3f0d642b8f824791abd4b82b30e9ebc2, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/e7a7cfd17ecb459c96bedbe9ba7c2113, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/5b091605999a4090b3791f81677fd963, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/543c721de2c5461aa233dd73225c2b53, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/20a1cc038125472c8ab5a0551d78915b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/b00b50c735504e4ab0c758cb13531e85, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/630e4f93ccee45759a647d6bd4311dec, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/128e3f7424af48a390926a071fe373d1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/6513528a0f0e4395bd601dc05bb9b48b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/31fbf39cc9fc4836a0ad7132e047a247, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/e2f264e1ba8342d7991638cd6f202ceb, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/877af8ac1213499daa1244f61f183b6f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/1b1edf3461b644ca84728291253ab40c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/bdfa8a3284d24510a007c4c117a4d870, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/dcf3c318c58245d79cf003d9dee70cdc] to archive 2024-12-09T01:42:54,898 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T01:42:54,899 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/2b319aceac1d472891fd0329c33735b6 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/2b319aceac1d472891fd0329c33735b6 2024-12-09T01:42:54,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/a32ab115f9e449968f9e1d2e45fe4034 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/a32ab115f9e449968f9e1d2e45fe4034 2024-12-09T01:42:54,900 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/b5b11f7000d5436ea9432668c4100c13 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/b5b11f7000d5436ea9432668c4100c13 2024-12-09T01:42:54,901 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/e92ad5d8c4d045c889ffaa9e565c6107 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/e92ad5d8c4d045c889ffaa9e565c6107 2024-12-09T01:42:54,902 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/5960d5885b2d4974b7d11b2f8657a409 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/5960d5885b2d4974b7d11b2f8657a409 2024-12-09T01:42:54,903 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/b7494de9f7444c8d83457c32f4245380 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/b7494de9f7444c8d83457c32f4245380 2024-12-09T01:42:54,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/6c9ceb64995049628f721870cefc7db0 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/6c9ceb64995049628f721870cefc7db0 2024-12-09T01:42:54,904 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/3ef536540d8a4c88a93816e0ac58d095 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/3ef536540d8a4c88a93816e0ac58d095 2024-12-09T01:42:54,905 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/bfb214fea9874901a6bec879a9a6ff36 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/bfb214fea9874901a6bec879a9a6ff36 2024-12-09T01:42:54,906 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/3f0d642b8f824791abd4b82b30e9ebc2 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/3f0d642b8f824791abd4b82b30e9ebc2 2024-12-09T01:42:54,907 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/e7a7cfd17ecb459c96bedbe9ba7c2113 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/e7a7cfd17ecb459c96bedbe9ba7c2113 2024-12-09T01:42:54,908 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/5b091605999a4090b3791f81677fd963 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/5b091605999a4090b3791f81677fd963 2024-12-09T01:42:54,909 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/543c721de2c5461aa233dd73225c2b53 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/543c721de2c5461aa233dd73225c2b53 2024-12-09T01:42:54,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/20a1cc038125472c8ab5a0551d78915b to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/20a1cc038125472c8ab5a0551d78915b 2024-12-09T01:42:54,910 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/b00b50c735504e4ab0c758cb13531e85 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/b00b50c735504e4ab0c758cb13531e85 2024-12-09T01:42:54,911 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/630e4f93ccee45759a647d6bd4311dec to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/630e4f93ccee45759a647d6bd4311dec 2024-12-09T01:42:54,912 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/128e3f7424af48a390926a071fe373d1 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/128e3f7424af48a390926a071fe373d1 2024-12-09T01:42:54,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/6513528a0f0e4395bd601dc05bb9b48b to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/6513528a0f0e4395bd601dc05bb9b48b 2024-12-09T01:42:54,913 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/31fbf39cc9fc4836a0ad7132e047a247 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/31fbf39cc9fc4836a0ad7132e047a247 2024-12-09T01:42:54,914 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/e2f264e1ba8342d7991638cd6f202ceb to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/e2f264e1ba8342d7991638cd6f202ceb 2024-12-09T01:42:54,915 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/877af8ac1213499daa1244f61f183b6f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/877af8ac1213499daa1244f61f183b6f 2024-12-09T01:42:54,916 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/1b1edf3461b644ca84728291253ab40c to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/1b1edf3461b644ca84728291253ab40c 2024-12-09T01:42:54,917 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/bdfa8a3284d24510a007c4c117a4d870 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/bdfa8a3284d24510a007c4c117a4d870 2024-12-09T01:42:54,918 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/dcf3c318c58245d79cf003d9dee70cdc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/dcf3c318c58245d79cf003d9dee70cdc 2024-12-09T01:42:54,919 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/0cbcb35c0e804541b46713e0f5ec47c0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/62f333f2217d4df99f83f320a034fda4, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/9d5ad637ce5c421f92dda57e61e61d27, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/959ec43b49554ca499e6de3b83d99f8e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/f8812d756ab94cb391bcbc7a9cde01a0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/fb9e0a3ae23d49e8b00d74fe275c2a4f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/05d43f07dd1d46b4b7851225897b5a61, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/3da68982540943708c24e175ef37a065, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/9edb812897a24b539bf687fe9110523d, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/7ee4ec2078944a91bfc9e053ba6d9f96, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/fc7be14357614621a0083f69123cbdab, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/076e125e2be84f7bb5834c8cc65772bb, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/db846a40393b4f40bd543bcfa312c81a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/b97e4bdb7e2148d79851cf939c9c32e6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/d6a74488203b404da1c04836b7ab8e16, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/ff35c79335e94125a7d157f38205fa1a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/b6e9c15aae864bf7be295c622196308d, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/a50a534ab46543809fd8642a2291ca12, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/b4441fbd664548b1b1d48b29565005d4, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/06525c87e4da486b8ba46f17c7bc8c6d, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/21c9c935ada1430ba52a8fb51451cac4, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/5c690a29412349b39df068c0f034277e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/d80510125ad042d0b975486bf2c1cd45, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/f5821fdd5bbb4792a63fc45f7670b550] to archive 2024-12-09T01:42:54,920 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T01:42:54,921 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/0cbcb35c0e804541b46713e0f5ec47c0 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/0cbcb35c0e804541b46713e0f5ec47c0 2024-12-09T01:42:54,922 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/62f333f2217d4df99f83f320a034fda4 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/62f333f2217d4df99f83f320a034fda4 2024-12-09T01:42:54,923 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/9d5ad637ce5c421f92dda57e61e61d27 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/9d5ad637ce5c421f92dda57e61e61d27 2024-12-09T01:42:54,924 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/959ec43b49554ca499e6de3b83d99f8e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/959ec43b49554ca499e6de3b83d99f8e 2024-12-09T01:42:54,924 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/f8812d756ab94cb391bcbc7a9cde01a0 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/f8812d756ab94cb391bcbc7a9cde01a0 2024-12-09T01:42:54,925 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/fb9e0a3ae23d49e8b00d74fe275c2a4f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/fb9e0a3ae23d49e8b00d74fe275c2a4f 2024-12-09T01:42:54,926 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/05d43f07dd1d46b4b7851225897b5a61 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/05d43f07dd1d46b4b7851225897b5a61 2024-12-09T01:42:54,927 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/3da68982540943708c24e175ef37a065 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/3da68982540943708c24e175ef37a065 2024-12-09T01:42:54,928 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/9edb812897a24b539bf687fe9110523d to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/9edb812897a24b539bf687fe9110523d 2024-12-09T01:42:54,929 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/7ee4ec2078944a91bfc9e053ba6d9f96 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/7ee4ec2078944a91bfc9e053ba6d9f96 2024-12-09T01:42:54,930 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/fc7be14357614621a0083f69123cbdab to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/fc7be14357614621a0083f69123cbdab 2024-12-09T01:42:54,931 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/076e125e2be84f7bb5834c8cc65772bb to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/076e125e2be84f7bb5834c8cc65772bb 2024-12-09T01:42:54,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/db846a40393b4f40bd543bcfa312c81a to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/db846a40393b4f40bd543bcfa312c81a 2024-12-09T01:42:54,932 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/b97e4bdb7e2148d79851cf939c9c32e6 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/b97e4bdb7e2148d79851cf939c9c32e6 2024-12-09T01:42:54,933 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/d6a74488203b404da1c04836b7ab8e16 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/d6a74488203b404da1c04836b7ab8e16 2024-12-09T01:42:54,934 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/ff35c79335e94125a7d157f38205fa1a to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/ff35c79335e94125a7d157f38205fa1a 2024-12-09T01:42:54,935 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/b6e9c15aae864bf7be295c622196308d to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/b6e9c15aae864bf7be295c622196308d 2024-12-09T01:42:54,936 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/a50a534ab46543809fd8642a2291ca12 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/a50a534ab46543809fd8642a2291ca12 2024-12-09T01:42:54,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/b4441fbd664548b1b1d48b29565005d4 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/b4441fbd664548b1b1d48b29565005d4 2024-12-09T01:42:54,937 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/06525c87e4da486b8ba46f17c7bc8c6d to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/06525c87e4da486b8ba46f17c7bc8c6d 2024-12-09T01:42:54,938 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/21c9c935ada1430ba52a8fb51451cac4 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/21c9c935ada1430ba52a8fb51451cac4 2024-12-09T01:42:54,939 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/5c690a29412349b39df068c0f034277e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/5c690a29412349b39df068c0f034277e 2024-12-09T01:42:54,940 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/d80510125ad042d0b975486bf2c1cd45 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/d80510125ad042d0b975486bf2c1cd45 2024-12-09T01:42:54,941 DEBUG [StoreCloser-TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/f5821fdd5bbb4792a63fc45f7670b550 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/f5821fdd5bbb4792a63fc45f7670b550 2024-12-09T01:42:54,945 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/recovered.edits/394.seqid, newMaxSeqId=394, maxSeqId=4 2024-12-09T01:42:54,945 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. 2024-12-09T01:42:54,945 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for 4b9a7812e88070d8f8c92ba7c915f7b9: Waiting for close lock at 1733708573222Running coprocessor pre-close hooks at 1733708573222Disabling compacts and flushes for region at 1733708573222Disabling writes for close at 1733708573222Obtaining lock to block concurrent updates at 1733708573222Preparing flush snapshotting stores in 4b9a7812e88070d8f8c92ba7c915f7b9 at 1733708573222Finished memstore snapshotting TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733708573223 (+1 ms)Flushing stores of TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9. at 1733708573223Flushing 4b9a7812e88070d8f8c92ba7c915f7b9/A: creating writer at 1733708573223Flushing mob file 4b9a7812e88070d8f8c92ba7c915f7b9/A: appending metadata at 1733708573227 (+4 ms)Flushing mob file 4b9a7812e88070d8f8c92ba7c915f7b9/A: closing flushed file at 1733708573227Flushing 4b9a7812e88070d8f8c92ba7c915f7b9/A: appending metadata at 1733708573635 (+408 ms)Flushing 4b9a7812e88070d8f8c92ba7c915f7b9/A: closing flushed file at 1733708573635Flushing 4b9a7812e88070d8f8c92ba7c915f7b9/B: creating writer at 1733708574042 (+407 ms)Flushing 4b9a7812e88070d8f8c92ba7c915f7b9/B: appending metadata at 1733708574044 (+2 ms)Flushing 4b9a7812e88070d8f8c92ba7c915f7b9/B: closing flushed file at 1733708574044Flushing 4b9a7812e88070d8f8c92ba7c915f7b9/C: creating writer at 1733708574452 (+408 ms)Flushing 4b9a7812e88070d8f8c92ba7c915f7b9/C: appending metadata at 1733708574454 (+2 ms)Flushing 4b9a7812e88070d8f8c92ba7c915f7b9/C: closing flushed file at 1733708574454Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2554e188: reopening flushed file at 1733708574861 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@422bf1b6: reopening flushed file at 1733708574865 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5646afad: reopening flushed file at 1733708574868 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 4b9a7812e88070d8f8c92ba7c915f7b9 in 1651ms, sequenceid=391, compaction requested=true at 1733708574873 (+5 ms)Writing region close event to WAL at 1733708574942 (+69 ms)Running coprocessor post-close hooks at 1733708574945 (+3 ms)Closed at 1733708574945 2024-12-09T01:42:54,947 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed 4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:54,947 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=4b9a7812e88070d8f8c92ba7c915f7b9, regionState=CLOSED 2024-12-09T01:42:54,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 because future has completed 2024-12-09T01:42:54,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-12-09T01:42:54,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure 4b9a7812e88070d8f8c92ba7c915f7b9, server=ef6f18c58dc9,38627,1733708453943 in 1.8790 sec 2024-12-09T01:42:54,952 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-12-09T01:42:54,952 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4b9a7812e88070d8f8c92ba7c915f7b9, UNASSIGN in 1.8860 sec 2024-12-09T01:42:54,954 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-12-09T01:42:54,954 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8900 sec 2024-12-09T01:42:54,955 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708574955"}]},"ts":"1733708574955"} 2024-12-09T01:42:54,956 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T01:42:54,956 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T01:42:54,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8980 sec 2024-12-09T01:42:55,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-12-09T01:42:55,204 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:55,205 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T01:42:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:55,206 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:55,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T01:42:55,207 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:55,209 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:55,211 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A, FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B, FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C, FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/recovered.edits] 2024-12-09T01:42:55,213 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/ce86752f9ed9444592528bbff7c0387f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/ce86752f9ed9444592528bbff7c0387f 2024-12-09T01:42:55,214 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/d0080082c42e4f6aa1f5a929f25a0b19 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/d0080082c42e4f6aa1f5a929f25a0b19 2024-12-09T01:42:55,215 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/e7a433d594b64eea9d0d947d84ab8312 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/A/e7a433d594b64eea9d0d947d84ab8312 2024-12-09T01:42:55,217 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/3359ff8457984d59b66863d3b4d6a942 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/3359ff8457984d59b66863d3b4d6a942 2024-12-09T01:42:55,217 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/d560d64361f54e3480d4521f4cdff21b to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/d560d64361f54e3480d4521f4cdff21b 2024-12-09T01:42:55,218 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/ef43493edea84c6b8c7d46e12fdf963b to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/B/ef43493edea84c6b8c7d46e12fdf963b 2024-12-09T01:42:55,220 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/8ef82a7288c0489f9d7a537a27e0a1cd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/8ef82a7288c0489f9d7a537a27e0a1cd 2024-12-09T01:42:55,221 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/ab7072efe6664ad1b47e53f41c2c8d4c to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/ab7072efe6664ad1b47e53f41c2c8d4c 2024-12-09T01:42:55,222 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/f08590770292469cb5c09ab4ac4c55eb to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/C/f08590770292469cb5c09ab4ac4c55eb 2024-12-09T01:42:55,224 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/recovered.edits/394.seqid to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9/recovered.edits/394.seqid 2024-12-09T01:42:55,225 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:55,225 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T01:42:55,225 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T01:42:55,226 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-09T01:42:55,228 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120912710fd3425a4ee6ac3fc871aec384ab_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120912710fd3425a4ee6ac3fc871aec384ab_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:55,229 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120921ea7c7a5d30417993b57efcf6670be7_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120921ea7c7a5d30417993b57efcf6670be7_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:55,230 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092211944569794335918489b47a1bdec0_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092211944569794335918489b47a1bdec0_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:55,230 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120925e920c78de6452787b110a4fc7bf7b5_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120925e920c78de6452787b110a4fc7bf7b5_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:55,231 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120937932c0ddee1445c9f0b0accd3eff474_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120937932c0ddee1445c9f0b0accd3eff474_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:55,232 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094b97cf7ac8434c309817161793cabe25_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412094b97cf7ac8434c309817161793cabe25_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:55,233 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120954be52780a7148208ff460073793593a_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120954be52780a7148208ff460073793593a_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:55,234 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120965a4a9fb16514c109e1eb3328d8abe1f_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120965a4a9fb16514c109e1eb3328d8abe1f_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:55,235 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097a3728f3593040f1add0b7491fa4af28_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412097a3728f3593040f1add0b7491fa4af28_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:55,236 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120984ffbc99d00d49f696bbe9e08364e304_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120984ffbc99d00d49f696bbe9e08364e304_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:55,237 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120988bdfff3ec9b4068a0e29a239e791ba1_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120988bdfff3ec9b4068a0e29a239e791ba1_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:55,237 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209910c02e62a3a44b19e1e7d484d2d417c_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209910c02e62a3a44b19e1e7d484d2d417c_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:55,238 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209aa820cd4d4d047fa8cf4d47097f302cc_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209aa820cd4d4d047fa8cf4d47097f302cc_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:55,239 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209adb74fd30cc249458a42a68c386853e7_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209adb74fd30cc249458a42a68c386853e7_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:55,240 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b2d2d6923ea84376b486e0fca8fdec05_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b2d2d6923ea84376b486e0fca8fdec05_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:55,241 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c8cd686929b84c9bb3a38455c9ea0b08_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c8cd686929b84c9bb3a38455c9ea0b08_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:55,242 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d00f74c9984a47e88013de13fd78facf_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d00f74c9984a47e88013de13fd78facf_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:55,243 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d58fbccac1d0472bb661c3eec4de567b_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209d58fbccac1d0472bb661c3eec4de567b_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:55,244 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209eb0a447a44204e3ea11f1f25a78aa016_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209eb0a447a44204e3ea11f1f25a78aa016_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:55,245 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ff99d5ae42a84238ab8438aa70d35bd2_4b9a7812e88070d8f8c92ba7c915f7b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209ff99d5ae42a84238ab8438aa70d35bd2_4b9a7812e88070d8f8c92ba7c915f7b9 2024-12-09T01:42:55,245 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T01:42:55,247 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:55,249 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T01:42:55,251 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T01:42:55,252 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:55,252 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T01:42:55,252 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733708575252"}]},"ts":"9223372036854775807"} 2024-12-09T01:42:55,254 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T01:42:55,254 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4b9a7812e88070d8f8c92ba7c915f7b9, NAME => 'TestAcidGuarantees,,1733708546885.4b9a7812e88070d8f8c92ba7c915f7b9.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T01:42:55,254 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T01:42:55,254 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733708575254"}]},"ts":"9223372036854775807"} 2024-12-09T01:42:55,255 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T01:42:55,256 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:55,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 51 msec 2024-12-09T01:42:55,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-09T01:42:55,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T01:42:55,314 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:55,323 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 250) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1484270681_22 at /127.0.0.1:33038 [Waiting for operation #235] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1484270681_22 at /127.0.0.1:39654 [Waiting for operation #286] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1866703801_22 at /127.0.0.1:32918 [Waiting for operation #609] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1866703801_22 at /127.0.0.1:33042 [Waiting for operation #227] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=594 (was 619), ProcessCount=11 (was 11), AvailableMemoryMB=11225 (was 11835) 2024-12-09T01:42:55,332 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=594, ProcessCount=11, AvailableMemoryMB=11224 2024-12-09T01:42:55,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T01:42:55,333 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:42:55,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T01:42:55,335 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T01:42:55,335 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:55,335 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-12-09T01:42:55,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-09T01:42:55,336 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T01:42:55,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742305_1481 (size=1061) 2024-12-09T01:42:55,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-09T01:42:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-09T01:42:55,742 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb 2024-12-09T01:42:55,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742306_1482 (size=53) 2024-12-09T01:42:55,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-09T01:42:56,148 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:42:56,148 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f66467034fd235041292f78356d98c07, disabling compactions & flushes 2024-12-09T01:42:56,148 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:56,148 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:56,148 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. after waiting 0 ms 2024-12-09T01:42:56,148 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:56,148 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:56,148 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f66467034fd235041292f78356d98c07: Waiting for close lock at 1733708576148Disabling compacts and flushes for region at 1733708576148Disabling writes for close at 1733708576148Writing region close event to WAL at 1733708576148Closed at 1733708576148 2024-12-09T01:42:56,149 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T01:42:56,149 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733708576149"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733708576149"}]},"ts":"1733708576149"} 2024-12-09T01:42:56,151 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T01:42:56,151 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T01:42:56,151 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708576151"}]},"ts":"1733708576151"} 2024-12-09T01:42:56,152 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T01:42:56,153 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f66467034fd235041292f78356d98c07, ASSIGN}] 2024-12-09T01:42:56,153 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f66467034fd235041292f78356d98c07, ASSIGN 2024-12-09T01:42:56,154 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f66467034fd235041292f78356d98c07, ASSIGN; state=OFFLINE, location=ef6f18c58dc9,38627,1733708453943; forceNewPlan=false, retain=false 2024-12-09T01:42:56,305 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=f66467034fd235041292f78356d98c07, regionState=OPENING, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:56,306 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f66467034fd235041292f78356d98c07, ASSIGN because future has completed 2024-12-09T01:42:56,306 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943}] 2024-12-09T01:42:56,461 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:56,461 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:42:56,462 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f66467034fd235041292f78356d98c07 2024-12-09T01:42:56,462 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:42:56,462 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for f66467034fd235041292f78356d98c07 2024-12-09T01:42:56,462 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for f66467034fd235041292f78356d98c07 2024-12-09T01:42:56,463 INFO [StoreOpener-f66467034fd235041292f78356d98c07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f66467034fd235041292f78356d98c07 2024-12-09T01:42:56,464 INFO [StoreOpener-f66467034fd235041292f78356d98c07-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:42:56,464 INFO [StoreOpener-f66467034fd235041292f78356d98c07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f66467034fd235041292f78356d98c07 columnFamilyName A 2024-12-09T01:42:56,464 DEBUG [StoreOpener-f66467034fd235041292f78356d98c07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:56,465 INFO [StoreOpener-f66467034fd235041292f78356d98c07-1 {}] regionserver.HStore(327): Store=f66467034fd235041292f78356d98c07/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:42:56,465 INFO [StoreOpener-f66467034fd235041292f78356d98c07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f66467034fd235041292f78356d98c07 2024-12-09T01:42:56,466 INFO [StoreOpener-f66467034fd235041292f78356d98c07-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:42:56,466 INFO [StoreOpener-f66467034fd235041292f78356d98c07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f66467034fd235041292f78356d98c07 columnFamilyName B 2024-12-09T01:42:56,466 DEBUG [StoreOpener-f66467034fd235041292f78356d98c07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:56,466 INFO [StoreOpener-f66467034fd235041292f78356d98c07-1 {}] regionserver.HStore(327): Store=f66467034fd235041292f78356d98c07/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:42:56,466 INFO [StoreOpener-f66467034fd235041292f78356d98c07-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f66467034fd235041292f78356d98c07 2024-12-09T01:42:56,467 INFO [StoreOpener-f66467034fd235041292f78356d98c07-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:42:56,467 INFO [StoreOpener-f66467034fd235041292f78356d98c07-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f66467034fd235041292f78356d98c07 columnFamilyName C 2024-12-09T01:42:56,467 DEBUG [StoreOpener-f66467034fd235041292f78356d98c07-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:42:56,467 INFO [StoreOpener-f66467034fd235041292f78356d98c07-1 {}] regionserver.HStore(327): Store=f66467034fd235041292f78356d98c07/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:42:56,467 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:56,468 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for f66467034fd235041292f78356d98c07 2024-12-09T01:42:56,468 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07 2024-12-09T01:42:56,468 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07 2024-12-09T01:42:56,469 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for f66467034fd235041292f78356d98c07 2024-12-09T01:42:56,469 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for f66467034fd235041292f78356d98c07 2024-12-09T01:42:56,470 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T01:42:56,470 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for f66467034fd235041292f78356d98c07 2024-12-09T01:42:56,472 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:42:56,472 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened f66467034fd235041292f78356d98c07; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69568525, jitterRate=0.03665180504322052}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T01:42:56,472 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f66467034fd235041292f78356d98c07 2024-12-09T01:42:56,473 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for f66467034fd235041292f78356d98c07: Running coprocessor pre-open hook at 1733708576462Writing region info on filesystem at 1733708576462Initializing all the Stores at 1733708576463 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708576463Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708576463Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708576463Cleaning up temporary data from old regions at 1733708576469 (+6 ms)Running coprocessor post-open hooks at 1733708576472 (+3 ms)Region opened successfully at 1733708576473 (+1 ms) 2024-12-09T01:42:56,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-09T01:42:56,474 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., pid=120, masterSystemTime=1733708576458 2024-12-09T01:42:56,475 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:56,475 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:56,476 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=f66467034fd235041292f78356d98c07, regionState=OPEN, openSeqNum=2, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:56,477 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 because future has completed 2024-12-09T01:42:56,479 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-09T01:42:56,479 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 in 172 msec 2024-12-09T01:42:56,480 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-12-09T01:42:56,480 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f66467034fd235041292f78356d98c07, ASSIGN in 326 msec 2024-12-09T01:42:56,481 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T01:42:56,481 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708576481"}]},"ts":"1733708576481"} 2024-12-09T01:42:56,482 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T01:42:56,483 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T01:42:56,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1500 sec 2024-12-09T01:42:56,513 WARN [regionserver/ef6f18c58dc9:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 5, running: 1 2024-12-09T01:42:57,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-12-09T01:42:57,484 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:57,484 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a19ac61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,485 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:57,485 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:57,486 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:57,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:57,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:57,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f3b5a89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:57,486 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:57,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:57,487 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59760, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:57,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64954991, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,488 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:57,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:57,489 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:57,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:57,490 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T01:42:57,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is ef6f18c58dc9,37995,1733708453073 2024-12-09T01:42:57,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7805b862 2024-12-09T01:42:57,491 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T01:42:57,492 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59770, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T01:42:57,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3adc9e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,493 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:57,493 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:57,493 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:57,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:57,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ab660bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:57,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:57,494 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e3e3014, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19a3b235, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,495 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:57,495 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:57,496 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:57,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:57,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:57,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@216a8268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:57,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:57,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:57,497 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:57,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c7f2af7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@720699be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,498 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:57,498 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:57,499 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78fd5ab1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:57,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:57,500 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:57,500 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59816, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:57,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@523c399a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78e2c23d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,501 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:57,501 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:57,501 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@847abff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:57,502 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:57,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18f696f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,503 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1931bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,503 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:57,503 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:57,504 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5daa31ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:57,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:57,505 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59870, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:57,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7213c72d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1589e05f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,506 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:57,506 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:57,506 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:57,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:57,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:57,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28671c7d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:57,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:57,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:57,507 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:57,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2923953c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,508 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e46d68a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,508 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:57,508 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:57,509 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@414517f0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:57,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:57,510 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59896, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:57,510 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ad9a4c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@127790d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,511 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:57,511 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:57,511 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:57,511 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:57,511 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:57,512 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3728be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,512 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:57,512 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:57,512 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:57,512 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:57,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2454d67f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2461d367, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,513 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:57,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:57,514 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:57,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:57,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:57,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28bebb27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:57,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:57,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:57,515 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59934, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:57,516 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d0fdf0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12a13cd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,516 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:42:57,516 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:42:57,517 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:42:57,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:42:57,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:42:57,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b97d51c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:42:57,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:42:57,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:42:57,518 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59948, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:42:57,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e638d1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:42:57,519 DEBUG [Thread-2154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:57,519 DEBUG [Thread-2148 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:57,519 DEBUG [Thread-2165 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:57,519 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:42:57,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:57,520 DEBUG [Thread-2157 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:57,520 DEBUG [Thread-2159 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:57,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:57,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:57,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:57,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:57,520 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:57,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:57,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:57,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:57,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:57,521 DEBUG [Thread-2163 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:57,522 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:57,522 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:57,522 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:57,523 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:57,523 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:57,523 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:57,523 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:57,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-12-09T01:42:57,524 DEBUG [Thread-2161 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:57,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T01:42:57,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:42:57,525 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:42:57,525 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:57,526 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:57,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:42:57,526 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:42:57,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:42:57,526 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:42:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:42:57,527 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:57,527 DEBUG [Thread-2150 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:57,527 DEBUG [Thread-2146 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:57,528 DEBUG [Thread-2146 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:57,529 DEBUG [Thread-2146 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:57,529 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:42:57,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:42:57,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:42:57,530 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:57,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:57,530 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:57,531 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:57,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:42:57,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:42:57,538 DEBUG [Thread-2152 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:42:57,540 DEBUG [Thread-2152 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:42:57,540 DEBUG [Thread-2152 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:42:57,541 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:42:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:42:57,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:42:57,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:42:57,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:57,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:42:57,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:57,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:42:57,541 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:57,542 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2] 2024-12-09T01:42:57,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/41692d84e39b4872a0801b4a3a224b91 is 50, key is test_row_0/A:col10/1733708577527/Put/seqid=0 2024-12-09T01:42:57,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742307_1483 (size=12001) 2024-12-09T01:42:57,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/41692d84e39b4872a0801b4a3a224b91 2024-12-09T01:42:57,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708577583, max timestamp=1733708577606Num uniques 859; 2024-12-09T01:42:57,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:42:57,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708577583, max timestamp=1733708577606Num uniques 859; 2024-12-09T01:42:57,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:42:57,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708577583, max timestamp=1733708577606Num uniques 859; 2024-12-09T01:42:57,611 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:42:57,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T01:42:57,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/8c6f6dfa0beb46239a609eebcf3cf6d1 is 50, key is test_row_0/B:col10/1733708577527/Put/seqid=0 2024-12-09T01:42:57,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742308_1484 (size=12001) 2024-12-09T01:42:57,647 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/8c6f6dfa0beb46239a609eebcf3cf6d1 2024-12-09T01:42:57,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:57,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708637649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:57,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:57,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:57,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708637651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:57,664 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:57,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/239f8827e57c46b89c932ee63daa8e61 is 50, key is test_row_0/C:col10/1733708577527/Put/seqid=0 2024-12-09T01:42:57,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742309_1485 (size=12001) 2024-12-09T01:42:57,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708637652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:57,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:57,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:57,677 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:57,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:57,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708637664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:57,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:57,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708637664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:57,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:57,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:57,681 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:57,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:57,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:57,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:57,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T01:42:57,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:57,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:42:57,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:57,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:57,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:57,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:57,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:57,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708637773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:57,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:57,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:57,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708637773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:57,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:57,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:57,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708637783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:57,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:57,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708637784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:57,793 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:57,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:57,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708637784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:57,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:57,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T01:42:57,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:57,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:42:57,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:57,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:57,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:57,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:57,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T01:42:57,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T01:42:57,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:57,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:42:57,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:57,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:57,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:57,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:58,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708637993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708637993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,004 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708638004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708638004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,015 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708638004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/239f8827e57c46b89c932ee63daa8e61 2024-12-09T01:42:58,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/41692d84e39b4872a0801b4a3a224b91 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/41692d84e39b4872a0801b4a3a224b91 2024-12-09T01:42:58,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/41692d84e39b4872a0801b4a3a224b91, entries=150, sequenceid=11, filesize=11.7 K 2024-12-09T01:42:58,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/8c6f6dfa0beb46239a609eebcf3cf6d1 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/8c6f6dfa0beb46239a609eebcf3cf6d1 2024-12-09T01:42:58,086 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/8c6f6dfa0beb46239a609eebcf3cf6d1, entries=150, sequenceid=11, filesize=11.7 K 2024-12-09T01:42:58,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/239f8827e57c46b89c932ee63daa8e61 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/239f8827e57c46b89c932ee63daa8e61 2024-12-09T01:42:58,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/239f8827e57c46b89c932ee63daa8e61, entries=150, sequenceid=11, filesize=11.7 K 2024-12-09T01:42:58,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for f66467034fd235041292f78356d98c07 in 550ms, sequenceid=11, compaction requested=false 2024-12-09T01:42:58,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:42:58,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-12-09T01:42:58,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:58,147 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-09T01:42:58,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:42:58,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:58,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:42:58,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:58,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:42:58,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:58,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/bbe1b093667b42eaaf14b7522008f083 is 50, key is test_row_0/A:col10/1733708577661/Put/seqid=0 2024-12-09T01:42:58,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T01:42:58,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742310_1486 (size=12001) 2024-12-09T01:42:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:42:58,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:42:58,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708638325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,336 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708638333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708638333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708638335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708638336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708638443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708638444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708638444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708638453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708638453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,559 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/bbe1b093667b42eaaf14b7522008f083 2024-12-09T01:42:58,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/4140f8b812a543efa1f6061cc9886368 is 50, key is test_row_0/B:col10/1733708577661/Put/seqid=0 2024-12-09T01:42:58,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742311_1487 (size=12001) 2024-12-09T01:42:58,577 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/4140f8b812a543efa1f6061cc9886368 2024-12-09T01:42:58,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/b67c2ff4f6ca447abb9bf81709d278ff is 50, key is test_row_0/C:col10/1733708577661/Put/seqid=0 2024-12-09T01:42:58,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742312_1488 (size=12001) 2024-12-09T01:42:58,599 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/b67c2ff4f6ca447abb9bf81709d278ff 2024-12-09T01:42:58,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/bbe1b093667b42eaaf14b7522008f083 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/bbe1b093667b42eaaf14b7522008f083 2024-12-09T01:42:58,607 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/bbe1b093667b42eaaf14b7522008f083, entries=150, sequenceid=48, filesize=11.7 K 2024-12-09T01:42:58,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/4140f8b812a543efa1f6061cc9886368 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/4140f8b812a543efa1f6061cc9886368 2024-12-09T01:42:58,611 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/4140f8b812a543efa1f6061cc9886368, entries=150, sequenceid=48, filesize=11.7 K 2024-12-09T01:42:58,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/b67c2ff4f6ca447abb9bf81709d278ff as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/b67c2ff4f6ca447abb9bf81709d278ff 2024-12-09T01:42:58,618 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/b67c2ff4f6ca447abb9bf81709d278ff, entries=150, sequenceid=48, filesize=11.7 K 2024-12-09T01:42:58,619 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for f66467034fd235041292f78356d98c07 in 472ms, sequenceid=48, compaction requested=false 2024-12-09T01:42:58,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:42:58,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:58,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-09T01:42:58,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-09T01:42:58,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-09T01:42:58,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0920 sec 2024-12-09T01:42:58,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.1060 sec 2024-12-09T01:42:58,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:42:58,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:42:58,657 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:42:58,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:58,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:42:58,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:58,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:42:58,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:42:58,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-09T01:42:58,663 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:42:58,664 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:42:58,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-12-09T01:42:58,665 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/158a50d6ed954ea8973ca746c7d85dc9 is 50, key is test_row_0/A:col10/1733708578656/Put/seqid=0 2024-12-09T01:42:58,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T01:42:58,666 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:42:58,667 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:42:58,667 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:42:58,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742313_1489 (size=14341) 2024-12-09T01:42:58,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708578702, max timestamp=1733708578716Num uniques 859; 2024-12-09T01:42:58,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708578702, max timestamp=1733708578716Num uniques 859; 2024-12-09T01:42:58,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:42:58,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:42:58,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708578702, max timestamp=1733708578716Num uniques 859; 2024-12-09T01:42:58,720 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:42:58,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708638757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708638758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T01:42:58,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708638759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708638759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708638760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,780 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T01:42:58,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:58,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:42:58,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:58,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:58,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:58,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708638863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,868 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708638873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708638884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708638884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708638884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:58,900 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:58,973 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T01:42:58,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:58,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:42:58,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:58,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:58,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:58,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:58,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T01:42:59,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/158a50d6ed954ea8973ca746c7d85dc9 2024-12-09T01:42:59,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/3ce7f72c851e494aa721d4c3f177579f is 50, key is test_row_0/B:col10/1733708578656/Put/seqid=0 2024-12-09T01:42:59,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742314_1490 (size=12001) 2024-12-09T01:42:59,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:59,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708639074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:59,090 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,090 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,090 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:59,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:59,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708639093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,105 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:59,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:59,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708639103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708639103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:59,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708639103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:59,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T01:42:59,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:59,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:42:59,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:59,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:59,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:59,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:59,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T01:42:59,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:59,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:42:59,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:59,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:59,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:59,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T01:42:59,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708639394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:59,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,401 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:59,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:59,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708639413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:59,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:59,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T01:42:59,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:59,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:42:59,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:59,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:59,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:59,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708639423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:59,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:59,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708639423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:59,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,435 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:59,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:59,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,435 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708639424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:59,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,436 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:59,485 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/3ce7f72c851e494aa721d4c3f177579f 2024-12-09T01:42:59,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/9ea72d89aa4d472e8e807e21d5888f4a is 50, key is test_row_0/C:col10/1733708578656/Put/seqid=0 2024-12-09T01:42:59,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742315_1491 (size=12001) 2024-12-09T01:42:59,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T01:42:59,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:59,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:42:59,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:59,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:59,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:59,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:59,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T01:42:59,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:59,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:42:59,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:59,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:59,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:59,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:59,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T01:42:59,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T01:42:59,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:59,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:42:59,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:59,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:59,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:59,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:42:59,898 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/9ea72d89aa4d472e8e807e21d5888f4a 2024-12-09T01:42:59,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/158a50d6ed954ea8973ca746c7d85dc9 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/158a50d6ed954ea8973ca746c7d85dc9 2024-12-09T01:42:59,905 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/158a50d6ed954ea8973ca746c7d85dc9, entries=200, sequenceid=58, filesize=14.0 K 2024-12-09T01:42:59,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/3ce7f72c851e494aa721d4c3f177579f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/3ce7f72c851e494aa721d4c3f177579f 2024-12-09T01:42:59,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/3ce7f72c851e494aa721d4c3f177579f, entries=150, sequenceid=58, filesize=11.7 K 2024-12-09T01:42:59,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/9ea72d89aa4d472e8e807e21d5888f4a as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/9ea72d89aa4d472e8e807e21d5888f4a 2024-12-09T01:42:59,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/9ea72d89aa4d472e8e807e21d5888f4a, entries=150, sequenceid=58, filesize=11.7 K 2024-12-09T01:42:59,914 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f66467034fd235041292f78356d98c07 in 1257ms, sequenceid=58, compaction requested=true 2024-12-09T01:42:59,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:42:59,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:42:59,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:59,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:42:59,914 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:59,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:59,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:42:59,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:59,914 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:59,915 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:59,915 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:59,915 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/A is initiating minor compaction (all files) 2024-12-09T01:42:59,915 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/B is initiating minor compaction (all files) 2024-12-09T01:42:59,915 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/A in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:59,915 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/B in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:59,915 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/41692d84e39b4872a0801b4a3a224b91, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/bbe1b093667b42eaaf14b7522008f083, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/158a50d6ed954ea8973ca746c7d85dc9] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=37.4 K 2024-12-09T01:42:59,915 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/8c6f6dfa0beb46239a609eebcf3cf6d1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/4140f8b812a543efa1f6061cc9886368, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/3ce7f72c851e494aa721d4c3f177579f] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=35.2 K 2024-12-09T01:42:59,915 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41692d84e39b4872a0801b4a3a224b91, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733708577527 2024-12-09T01:42:59,915 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c6f6dfa0beb46239a609eebcf3cf6d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733708577527 2024-12-09T01:42:59,916 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbe1b093667b42eaaf14b7522008f083, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733708577648 2024-12-09T01:42:59,916 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 4140f8b812a543efa1f6061cc9886368, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733708577648 2024-12-09T01:42:59,916 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 158a50d6ed954ea8973ca746c7d85dc9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733708578331 2024-12-09T01:42:59,916 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ce7f72c851e494aa721d4c3f177579f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733708578331 2024-12-09T01:42:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:42:59,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-12-09T01:42:59,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:42:59,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:59,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:42:59,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:59,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:42:59,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:42:59,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/3d2a14e9124145de9987b4359abe7805 is 50, key is test_row_0/A:col10/1733708578757/Put/seqid=0 2024-12-09T01:42:59,926 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#B#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:59,927 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/2f8b43fd2565419bb0594e0163428a57 is 50, key is test_row_0/B:col10/1733708578656/Put/seqid=0 2024-12-09T01:42:59,929 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#A#compaction#418 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:59,930 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/c972ed7baeb04ccfbdf8d4d3059f81f0 is 50, key is test_row_0/A:col10/1733708578656/Put/seqid=0 2024-12-09T01:42:59,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742316_1492 (size=14341) 2024-12-09T01:42:59,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742317_1493 (size=12104) 2024-12-09T01:42:59,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/3d2a14e9124145de9987b4359abe7805 2024-12-09T01:42:59,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742318_1494 (size=12104) 2024-12-09T01:42:59,952 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/c972ed7baeb04ccfbdf8d4d3059f81f0 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/c972ed7baeb04ccfbdf8d4d3059f81f0 2024-12-09T01:42:59,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/a12f7098f3b0465895703717aca8ca22 is 50, key is test_row_0/B:col10/1733708578757/Put/seqid=0 2024-12-09T01:42:59,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708639944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:59,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:59,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:59,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708639948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,958 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:59,961 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f66467034fd235041292f78356d98c07/A of f66467034fd235041292f78356d98c07 into c972ed7baeb04ccfbdf8d4d3059f81f0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:59,961 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:42:59,961 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/A, priority=13, startTime=1733708579914; duration=0sec 2024-12-09T01:42:59,961 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:42:59,961 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:A 2024-12-09T01:42:59,961 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:42:59,962 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:42:59,962 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/C is initiating minor compaction (all files) 2024-12-09T01:42:59,962 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/C in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:42:59,962 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/239f8827e57c46b89c932ee63daa8e61, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/b67c2ff4f6ca447abb9bf81709d278ff, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/9ea72d89aa4d472e8e807e21d5888f4a] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=35.2 K 2024-12-09T01:42:59,963 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 239f8827e57c46b89c932ee63daa8e61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1733708577527 2024-12-09T01:42:59,963 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting b67c2ff4f6ca447abb9bf81709d278ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1733708577648 2024-12-09T01:42:59,963 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ea72d89aa4d472e8e807e21d5888f4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733708578331 2024-12-09T01:42:59,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742319_1495 (size=12001) 2024-12-09T01:42:59,972 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#C#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:42:59,972 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/2176e3fc19144167a33c449aed55de40 is 50, key is test_row_0/C:col10/1733708578656/Put/seqid=0 2024-12-09T01:42:59,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/a12f7098f3b0465895703717aca8ca22 2024-12-09T01:42:59,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708639950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:59,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708639951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:59,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:42:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708639956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:42:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:59,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:42:59,977 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:42:59,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742320_1496 (size=12104) 2024-12-09T01:42:59,990 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/2176e3fc19144167a33c449aed55de40 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/2176e3fc19144167a33c449aed55de40 2024-12-09T01:42:59,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/6d5ed1e31b1145b1891a4664d3e3f451 is 50, key is test_row_0/C:col10/1733708578757/Put/seqid=0 2024-12-09T01:42:59,995 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f66467034fd235041292f78356d98c07/C of f66467034fd235041292f78356d98c07 into 2176e3fc19144167a33c449aed55de40(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:42:59,995 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:42:59,995 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/C, priority=13, startTime=1733708579914; duration=0sec 2024-12-09T01:42:59,995 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:42:59,995 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:C 2024-12-09T01:43:00,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742321_1497 (size=12001) 2024-12-09T01:43:00,046 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T01:43:00,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:00,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:00,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:00,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:00,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:00,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:00,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708640063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:00,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,075 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:00,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:00,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708640084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:00,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:00,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708640084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:00,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,091 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:00,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:00,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708640084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:00,091 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:00,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,091 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:00,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T01:43:00,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:00,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:00,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:00,200 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:00,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:00,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708640283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:00,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,296 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:00,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708640294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708640294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:00,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708640294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:00,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:00,347 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/2f8b43fd2565419bb0594e0163428a57 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/2f8b43fd2565419bb0594e0163428a57 2024-12-09T01:43:00,352 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f66467034fd235041292f78356d98c07/B of f66467034fd235041292f78356d98c07 into 2f8b43fd2565419bb0594e0163428a57(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:00,352 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:00,352 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/B, priority=13, startTime=1733708579914; duration=0sec 2024-12-09T01:43:00,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T01:43:00,352 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:00,352 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:B 2024-12-09T01:43:00,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:00,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:00,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:00,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:00,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:00,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:00,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/6d5ed1e31b1145b1891a4664d3e3f451 2024-12-09T01:43:00,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/3d2a14e9124145de9987b4359abe7805 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/3d2a14e9124145de9987b4359abe7805 2024-12-09T01:43:00,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/3d2a14e9124145de9987b4359abe7805, entries=200, sequenceid=95, filesize=14.0 K 2024-12-09T01:43:00,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/a12f7098f3b0465895703717aca8ca22 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/a12f7098f3b0465895703717aca8ca22 2024-12-09T01:43:00,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/a12f7098f3b0465895703717aca8ca22, entries=150, sequenceid=95, filesize=11.7 K 2024-12-09T01:43:00,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/6d5ed1e31b1145b1891a4664d3e3f451 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/6d5ed1e31b1145b1891a4664d3e3f451 2024-12-09T01:43:00,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/6d5ed1e31b1145b1891a4664d3e3f451, entries=150, sequenceid=95, filesize=11.7 K 2024-12-09T01:43:00,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for f66467034fd235041292f78356d98c07 in 502ms, sequenceid=95, compaction requested=false 2024-12-09T01:43:00,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:00,469 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T01:43:00,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-12-09T01:43:00,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:00,509 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-09T01:43:00,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:00,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:00,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:00,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:00,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:00,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:00,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/3a856fa00abe4b729e18fff25da3bdb1 is 50, key is test_row_0/A:col10/1733708579951/Put/seqid=0 2024-12-09T01:43:00,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742322_1498 (size=12001) 2024-12-09T01:43:00,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:00,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:00,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708580634, max timestamp=1733708580663Num uniques 859; 2024-12-09T01:43:00,667 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:43:00,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708580634, max timestamp=1733708580663Num uniques 859; 2024-12-09T01:43:00,670 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:43:00,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708580634, max timestamp=1733708580663Num uniques 859; 2024-12-09T01:43:00,671 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:43:00,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708640731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:00,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:00,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733708580691, max timestamp=1733708580751Num uniques 1018; 2024-12-09T01:43:00,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733708580691, max timestamp=1733708580751Num uniques 1018; 2024-12-09T01:43:00,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-09T01:43:00,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=12268, suffix heap size=133840,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=268 2024-12-09T01:43:00,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=268, cellSize=12268, totalHeapSize=28112, min timestamp=1733708580691, max timestamp=1733708580751Num uniques 1068; 2024-12-09T01:43:00,757 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=12268, suffix heap size=139040,new segment heap size=28112 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=268 2024-12-09T01:43:00,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T01:43:00,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708640826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:00,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708640827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:00,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,843 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:00,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:00,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708640837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,844 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:00,876 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708640863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:00,922 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/3a856fa00abe4b729e18fff25da3bdb1 2024-12-09T01:43:00,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/88a4d95112834017ab9a9ba63d8d7579 is 50, key is test_row_0/B:col10/1733708579951/Put/seqid=0 2024-12-09T01:43:00,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742323_1499 (size=12001) 2024-12-09T01:43:00,939 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/88a4d95112834017ab9a9ba63d8d7579 2024-12-09T01:43:00,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/296f9f714a32496d8f64b0cd1fb8b2fe is 50, key is test_row_0/C:col10/1733708579951/Put/seqid=0 2024-12-09T01:43:00,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742324_1500 (size=12001) 2024-12-09T01:43:00,953 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/296f9f714a32496d8f64b0cd1fb8b2fe 2024-12-09T01:43:00,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/3a856fa00abe4b729e18fff25da3bdb1 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/3a856fa00abe4b729e18fff25da3bdb1 2024-12-09T01:43:00,967 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/3a856fa00abe4b729e18fff25da3bdb1, entries=150, sequenceid=107, filesize=11.7 K 2024-12-09T01:43:00,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:00,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/88a4d95112834017ab9a9ba63d8d7579 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/88a4d95112834017ab9a9ba63d8d7579 2024-12-09T01:43:00,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708640953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:00,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:00,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708640953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:00,968 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:00,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:00,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:00,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708640953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:00,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:00,969 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:00,973 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/88a4d95112834017ab9a9ba63d8d7579, entries=150, sequenceid=107, filesize=11.7 K 2024-12-09T01:43:00,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/296f9f714a32496d8f64b0cd1fb8b2fe as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/296f9f714a32496d8f64b0cd1fb8b2fe 2024-12-09T01:43:00,977 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/296f9f714a32496d8f64b0cd1fb8b2fe, entries=150, sequenceid=107, filesize=11.7 K 2024-12-09T01:43:00,978 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for f66467034fd235041292f78356d98c07 in 469ms, sequenceid=107, compaction requested=true 2024-12-09T01:43:00,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:00,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:00,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-09T01:43:00,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-12-09T01:43:00,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:00,986 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=147.60 KB heapSize=414.61 KB 2024-12-09T01:43:00,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:00,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:00,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:00,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:00,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:00,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:00,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-12-09T01:43:00,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3170 sec 2024-12-09T01:43:00,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.3240 sec 2024-12-09T01:43:00,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/e505587c817d40bb8e596ddbfaaaaf82 is 50, key is test_row_0/A:col10/1733708580985/Put/seqid=0 2024-12-09T01:43:00,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742325_1501 (size=14541) 2024-12-09T01:43:00,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/e505587c817d40bb8e596ddbfaaaaf82 2024-12-09T01:43:01,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/788a066d03a94575bc2d4a5b806e23e2 is 50, key is test_row_0/B:col10/1733708580985/Put/seqid=0 2024-12-09T01:43:01,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742326_1502 (size=12151) 2024-12-09T01:43:01,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:01,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708641039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,047 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:01,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:01,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708641083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:01,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:01,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708641154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:01,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:01,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708641173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:01,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:01,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708641173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:01,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,186 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:01,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:01,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708641173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:01,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:01,187 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,187 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,187 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:01,382 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:01,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708641373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:01,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,382 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:01,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/788a066d03a94575bc2d4a5b806e23e2 2024-12-09T01:43:01,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:01,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708641403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:01,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:01,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/4a02e4623c6943dcaab4d9c10f2d0de2 is 50, key is test_row_0/C:col10/1733708580985/Put/seqid=0 2024-12-09T01:43:01,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742327_1503 (size=12151) 2024-12-09T01:43:01,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:01,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708641494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:01,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:01,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,504 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:01,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708641494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:01,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:01,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708641494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:01,505 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:01,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,505 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:01,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:01,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708641693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:01,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:01,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:01,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=161 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/4a02e4623c6943dcaab4d9c10f2d0de2 2024-12-09T01:43:01,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/e505587c817d40bb8e596ddbfaaaaf82 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/e505587c817d40bb8e596ddbfaaaaf82 2024-12-09T01:43:01,832 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/e505587c817d40bb8e596ddbfaaaaf82, entries=200, sequenceid=161, filesize=14.2 K 2024-12-09T01:43:01,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/788a066d03a94575bc2d4a5b806e23e2 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/788a066d03a94575bc2d4a5b806e23e2 2024-12-09T01:43:01,836 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/788a066d03a94575bc2d4a5b806e23e2, entries=150, sequenceid=161, filesize=11.9 K 2024-12-09T01:43:01,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/4a02e4623c6943dcaab4d9c10f2d0de2 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/4a02e4623c6943dcaab4d9c10f2d0de2 2024-12-09T01:43:01,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/4a02e4623c6943dcaab4d9c10f2d0de2, entries=150, sequenceid=161, filesize=11.9 K 2024-12-09T01:43:01,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~414.56 KB/424512, currentSize=33.54 KB/34350 for f66467034fd235041292f78356d98c07 in 854ms, sequenceid=161, compaction requested=true 2024-12-09T01:43:01,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:01,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:43:01,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:01,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:43:01,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:01,841 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:43:01,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:43:01,841 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:01,841 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:43:01,842 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52987 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:43:01,842 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:43:01,842 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/A is initiating minor compaction (all files) 2024-12-09T01:43:01,842 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/B is initiating minor compaction (all files) 2024-12-09T01:43:01,842 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/A in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:01,842 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/B in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:01,842 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/c972ed7baeb04ccfbdf8d4d3059f81f0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/3d2a14e9124145de9987b4359abe7805, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/3a856fa00abe4b729e18fff25da3bdb1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/e505587c817d40bb8e596ddbfaaaaf82] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=51.7 K 2024-12-09T01:43:01,842 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/2f8b43fd2565419bb0594e0163428a57, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/a12f7098f3b0465895703717aca8ca22, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/88a4d95112834017ab9a9ba63d8d7579, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/788a066d03a94575bc2d4a5b806e23e2] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=47.1 K 2024-12-09T01:43:01,843 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f8b43fd2565419bb0594e0163428a57, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733708578331 2024-12-09T01:43:01,843 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting c972ed7baeb04ccfbdf8d4d3059f81f0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733708578331 2024-12-09T01:43:01,844 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting a12f7098f3b0465895703717aca8ca22, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733708578757 2024-12-09T01:43:01,844 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d2a14e9124145de9987b4359abe7805, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733708578757 2024-12-09T01:43:01,844 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88a4d95112834017ab9a9ba63d8d7579, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733708579948 2024-12-09T01:43:01,844 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a856fa00abe4b729e18fff25da3bdb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733708579948 2024-12-09T01:43:01,844 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting e505587c817d40bb8e596ddbfaaaaf82, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733708580803 2024-12-09T01:43:01,844 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 788a066d03a94575bc2d4a5b806e23e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733708580803 2024-12-09T01:43:01,859 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#A#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:01,860 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/c939a964d0cc4a76a8ed65814c9be080 is 50, key is test_row_0/A:col10/1733708580985/Put/seqid=0 2024-12-09T01:43:01,864 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#B#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:01,865 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/94141d697f204096a9fe4eb189db0046 is 50, key is test_row_0/B:col10/1733708580985/Put/seqid=0 2024-12-09T01:43:01,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742328_1504 (size=12391) 2024-12-09T01:43:01,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742329_1505 (size=12391) 2024-12-09T01:43:01,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:01,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:43:01,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:01,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:01,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:01,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:01,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:01,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:01,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/00c0c84d2c2541529fdd1e3d1e1a2296 is 50, key is test_row_0/A:col10/1733708581037/Put/seqid=0 2024-12-09T01:43:01,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742330_1506 (size=14541) 2024-12-09T01:43:02,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733708582022, max timestamp=1733708582041Num uniques 859; 2024-12-09T01:43:02,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T01:43:02,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733708582022, max timestamp=1733708582041Num uniques 859; 2024-12-09T01:43:02,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T01:43:02,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733708582022, max timestamp=1733708582041Num uniques 859; 2024-12-09T01:43:02,046 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T01:43:02,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708642096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:02,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:02,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708642107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:02,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:02,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:02,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708642109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:02,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:02,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708642110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:02,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,126 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:02,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:02,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708642204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:02,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:02,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:02,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708642223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:02,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:02,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708642223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:02,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:02,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,235 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:02,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:02,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708642234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:02,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:02,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708642234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:02,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,242 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:02,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:02,282 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/c939a964d0cc4a76a8ed65814c9be080 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/c939a964d0cc4a76a8ed65814c9be080 2024-12-09T01:43:02,286 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f66467034fd235041292f78356d98c07/A of f66467034fd235041292f78356d98c07 into c939a964d0cc4a76a8ed65814c9be080(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:02,286 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:02,286 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/A, priority=12, startTime=1733708581841; duration=0sec 2024-12-09T01:43:02,286 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:02,286 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:A 2024-12-09T01:43:02,286 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:43:02,288 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48257 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:43:02,288 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/C is initiating minor compaction (all files) 2024-12-09T01:43:02,288 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/C in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:02,288 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/2176e3fc19144167a33c449aed55de40, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/6d5ed1e31b1145b1891a4664d3e3f451, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/296f9f714a32496d8f64b0cd1fb8b2fe, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/4a02e4623c6943dcaab4d9c10f2d0de2] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=47.1 K 2024-12-09T01:43:02,288 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 2176e3fc19144167a33c449aed55de40, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1733708578331 2024-12-09T01:43:02,289 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d5ed1e31b1145b1891a4664d3e3f451, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1733708578757 2024-12-09T01:43:02,289 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 296f9f714a32496d8f64b0cd1fb8b2fe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1733708579948 2024-12-09T01:43:02,289 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a02e4623c6943dcaab4d9c10f2d0de2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733708580803 2024-12-09T01:43:02,290 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/94141d697f204096a9fe4eb189db0046 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/94141d697f204096a9fe4eb189db0046 2024-12-09T01:43:02,295 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f66467034fd235041292f78356d98c07/B of f66467034fd235041292f78356d98c07 into 94141d697f204096a9fe4eb189db0046(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:02,295 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:02,295 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/B, priority=12, startTime=1733708581841; duration=0sec 2024-12-09T01:43:02,295 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:02,295 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:B 2024-12-09T01:43:02,298 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#C#compaction#431 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:02,299 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/fc406365557445d6abe027e1202f4a98 is 50, key is test_row_0/C:col10/1733708580985/Put/seqid=0 2024-12-09T01:43:02,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742331_1507 (size=12391) 2024-12-09T01:43:02,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/00c0c84d2c2541529fdd1e3d1e1a2296 2024-12-09T01:43:02,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/dfbaba299beb452d8ab7257b2d898e59 is 50, key is test_row_0/B:col10/1733708581037/Put/seqid=0 2024-12-09T01:43:02,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742332_1508 (size=12151) 2024-12-09T01:43:02,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:02,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708642443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:02,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:02,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708642444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:02,453 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:02,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,453 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:02,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:02,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708642454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:02,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:02,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708642454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:02,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:02,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,467 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:02,710 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/fc406365557445d6abe027e1202f4a98 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/fc406365557445d6abe027e1202f4a98 2024-12-09T01:43:02,714 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f66467034fd235041292f78356d98c07/C of f66467034fd235041292f78356d98c07 into fc406365557445d6abe027e1202f4a98(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:02,714 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:02,714 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/C, priority=12, startTime=1733708581841; duration=0sec 2024-12-09T01:43:02,714 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:02,714 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:C 2024-12-09T01:43:02,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/dfbaba299beb452d8ab7257b2d898e59 2024-12-09T01:43:02,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708642764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:02,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:02,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708642764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:02,767 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/5d55e0c812aa40b984714c5379ba8a6a is 50, key is test_row_0/C:col10/1733708581037/Put/seqid=0 2024-12-09T01:43:02,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:02,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:02,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742333_1509 (size=12151) 2024-12-09T01:43:02,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:02,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708642774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:02,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:02,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708642774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:02,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:02,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:02,784 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-09T01:43:02,834 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:02,834 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:43:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-12-09T01:43:02,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T01:43:02,836 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:43:02,837 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:43:02,837 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:43:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T01:43:02,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T01:43:02,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:02,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:02,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:02,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:02,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:02,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:03,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T01:43:03,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:03,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:03,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:03,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:03,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:03,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T01:43:03,177 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/5d55e0c812aa40b984714c5379ba8a6a 2024-12-09T01:43:03,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/00c0c84d2c2541529fdd1e3d1e1a2296 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/00c0c84d2c2541529fdd1e3d1e1a2296 2024-12-09T01:43:03,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/00c0c84d2c2541529fdd1e3d1e1a2296, entries=200, sequenceid=171, filesize=14.2 K 2024-12-09T01:43:03,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/dfbaba299beb452d8ab7257b2d898e59 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/dfbaba299beb452d8ab7257b2d898e59 2024-12-09T01:43:03,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/dfbaba299beb452d8ab7257b2d898e59, entries=150, sequenceid=171, filesize=11.9 K 2024-12-09T01:43:03,207 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/5d55e0c812aa40b984714c5379ba8a6a as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/5d55e0c812aa40b984714c5379ba8a6a 2024-12-09T01:43:03,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/5d55e0c812aa40b984714c5379ba8a6a, entries=150, sequenceid=171, filesize=11.9 K 2024-12-09T01:43:03,224 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for f66467034fd235041292f78356d98c07 in 1291ms, sequenceid=171, compaction requested=false 2024-12-09T01:43:03,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:03,256 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-12-09T01:43:03,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:03,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:03,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:03,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:03,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:03,256 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:03,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/220499b8ac544d4288a35ad0851de9ca is 50, key is test_row_0/A:col10/1733708582094/Put/seqid=0 2024-12-09T01:43:03,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742334_1510 (size=14541) 2024-12-09T01:43:03,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/220499b8ac544d4288a35ad0851de9ca 2024-12-09T01:43:03,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/d1ea6ad0529240e2b6c9c06e3056490c is 50, key is test_row_0/B:col10/1733708582094/Put/seqid=0 2024-12-09T01:43:03,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742335_1511 (size=12151) 2024-12-09T01:43:03,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/d1ea6ad0529240e2b6c9c06e3056490c 2024-12-09T01:43:03,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T01:43:03,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:03,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:03,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:03,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:03,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:03,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:03,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:03,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708643299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:03,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708643299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:03,327 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/ec1c7f89c470422480f790cb99f97b1c is 50, key is test_row_0/C:col10/1733708582094/Put/seqid=0 2024-12-09T01:43:03,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708643312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:03,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:03,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708643315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:03,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,332 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:03,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:03,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:03,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708643317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:03,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742336_1512 (size=12151) 2024-12-09T01:43:03,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/ec1c7f89c470422480f790cb99f97b1c 2024-12-09T01:43:03,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/220499b8ac544d4288a35ad0851de9ca as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/220499b8ac544d4288a35ad0851de9ca 2024-12-09T01:43:03,402 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/220499b8ac544d4288a35ad0851de9ca, entries=200, sequenceid=210, filesize=14.2 K 2024-12-09T01:43:03,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/d1ea6ad0529240e2b6c9c06e3056490c as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/d1ea6ad0529240e2b6c9c06e3056490c 2024-12-09T01:43:03,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/d1ea6ad0529240e2b6c9c06e3056490c, entries=150, sequenceid=210, filesize=11.9 K 2024-12-09T01:43:03,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/ec1c7f89c470422480f790cb99f97b1c as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/ec1c7f89c470422480f790cb99f97b1c 2024-12-09T01:43:03,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/ec1c7f89c470422480f790cb99f97b1c, entries=150, sequenceid=210, filesize=11.9 K 2024-12-09T01:43:03,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=46.96 KB/48090 for f66467034fd235041292f78356d98c07 in 166ms, sequenceid=210, compaction requested=true 2024-12-09T01:43:03,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:03,423 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:03,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:43:03,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:03,423 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:03,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:43:03,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:03,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:43:03,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:03,424 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41473 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:03,424 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/A is initiating minor compaction (all files) 2024-12-09T01:43:03,424 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/A in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:03,424 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/c939a964d0cc4a76a8ed65814c9be080, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/00c0c84d2c2541529fdd1e3d1e1a2296, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/220499b8ac544d4288a35ad0851de9ca] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=40.5 K 2024-12-09T01:43:03,424 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:03,424 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/B is initiating minor compaction (all files) 2024-12-09T01:43:03,424 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/B in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:03,425 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/94141d697f204096a9fe4eb189db0046, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/dfbaba299beb452d8ab7257b2d898e59, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/d1ea6ad0529240e2b6c9c06e3056490c] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=35.8 K 2024-12-09T01:43:03,425 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting c939a964d0cc4a76a8ed65814c9be080, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733708580803 2024-12-09T01:43:03,425 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 94141d697f204096a9fe4eb189db0046, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733708580803 2024-12-09T01:43:03,426 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00c0c84d2c2541529fdd1e3d1e1a2296, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733708581030 2024-12-09T01:43:03,426 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting dfbaba299beb452d8ab7257b2d898e59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733708581037 2024-12-09T01:43:03,426 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 220499b8ac544d4288a35ad0851de9ca, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733708582094 2024-12-09T01:43:03,427 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting d1ea6ad0529240e2b6c9c06e3056490c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733708582094 2024-12-09T01:43:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:03,441 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T01:43:03,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:03,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:03,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:03,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:03,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:03,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:03,446 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#A#compaction#437 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:03,447 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/e826e9f3cb2f4006bf3048525fa5096f is 50, key is test_row_0/A:col10/1733708582094/Put/seqid=0 2024-12-09T01:43:03,450 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#B#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:03,450 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/d916e9b2008b4e80aac8582512917d10 is 50, key is test_row_0/B:col10/1733708582094/Put/seqid=0 2024-12-09T01:43:03,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T01:43:03,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:03,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:03,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:03,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:03,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:03,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:03,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T01:43:03,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/a6399a72822a48de8f3cffcf4c375125 is 50, key is test_row_0/A:col10/1733708583435/Put/seqid=0 2024-12-09T01:43:03,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742337_1513 (size=12493) 2024-12-09T01:43:03,488 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/e826e9f3cb2f4006bf3048525fa5096f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/e826e9f3cb2f4006bf3048525fa5096f 2024-12-09T01:43:03,497 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f66467034fd235041292f78356d98c07/A of f66467034fd235041292f78356d98c07 into e826e9f3cb2f4006bf3048525fa5096f(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:03,497 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:03,497 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/A, priority=13, startTime=1733708583422; duration=0sec 2024-12-09T01:43:03,498 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:03,498 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:A 2024-12-09T01:43:03,499 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:03,499 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36693 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:03,499 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/C is initiating minor compaction (all files) 2024-12-09T01:43:03,499 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/C in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:03,500 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/fc406365557445d6abe027e1202f4a98, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/5d55e0c812aa40b984714c5379ba8a6a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/ec1c7f89c470422480f790cb99f97b1c] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=35.8 K 2024-12-09T01:43:03,501 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc406365557445d6abe027e1202f4a98, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=161, earliestPutTs=1733708580803 2024-12-09T01:43:03,503 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d55e0c812aa40b984714c5379ba8a6a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1733708581037 2024-12-09T01:43:03,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742338_1514 (size=12493) 2024-12-09T01:43:03,506 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec1c7f89c470422480f790cb99f97b1c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733708582094 2024-12-09T01:43:03,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742339_1515 (size=16931) 2024-12-09T01:43:03,516 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#C#compaction#440 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:03,519 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/c82ec52a3c2c4d3894f9995fe7582909 is 50, key is test_row_0/C:col10/1733708582094/Put/seqid=0 2024-12-09T01:43:03,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/a6399a72822a48de8f3cffcf4c375125 2024-12-09T01:43:03,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:03,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708643535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:03,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,551 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:03,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:03,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708643537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:03,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,553 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:03,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742340_1516 (size=12493) 2024-12-09T01:43:03,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:03,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708643546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:03,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:03,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/b93de55e55fe44b88ba13d280b006bd3 is 50, key is test_row_0/B:col10/1733708583435/Put/seqid=0 2024-12-09T01:43:03,563 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/c82ec52a3c2c4d3894f9995fe7582909 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/c82ec52a3c2c4d3894f9995fe7582909 2024-12-09T01:43:03,567 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f66467034fd235041292f78356d98c07/C of f66467034fd235041292f78356d98c07 into c82ec52a3c2c4d3894f9995fe7582909(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:03,567 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:03,567 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/C, priority=13, startTime=1733708583423; duration=0sec 2024-12-09T01:43:03,567 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:03,567 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:C 2024-12-09T01:43:03,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:03,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708643548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:03,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:03,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:03,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708643550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:03,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742341_1517 (size=12151) 2024-12-09T01:43:03,582 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/b93de55e55fe44b88ba13d280b006bd3 2024-12-09T01:43:03,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/efdbbf24f55b42569ec758bd829db0d5 is 50, key is test_row_0/C:col10/1733708583435/Put/seqid=0 2024-12-09T01:43:03,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742342_1518 (size=12151) 2024-12-09T01:43:03,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T01:43:03,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:03,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:03,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:03,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:03,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:03,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:03,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708643654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:03,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:03,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708643663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:03,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:03,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708643663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:03,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:03,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,674 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:03,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708643673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:03,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:03,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708643673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:03,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:03,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:03,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T01:43:03,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:03,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:03,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:03,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:03,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:03,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:03,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:03,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708643874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:03,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,885 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:03,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:03,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708643883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:03,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708643883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,892 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:03,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:03,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708643894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:03,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:03,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:03,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708643894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:03,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:03,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:03,910 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/d916e9b2008b4e80aac8582512917d10 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/d916e9b2008b4e80aac8582512917d10 2024-12-09T01:43:03,915 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f66467034fd235041292f78356d98c07/B of f66467034fd235041292f78356d98c07 into d916e9b2008b4e80aac8582512917d10(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:03,915 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:03,915 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/B, priority=13, startTime=1733708583423; duration=0sec 2024-12-09T01:43:03,915 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:03,915 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:B 2024-12-09T01:43:03,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T01:43:03,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:03,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:03,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:03,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:03,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:03,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:03,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T01:43:04,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=222 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/efdbbf24f55b42569ec758bd829db0d5 2024-12-09T01:43:04,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/a6399a72822a48de8f3cffcf4c375125 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/a6399a72822a48de8f3cffcf4c375125 2024-12-09T01:43:04,018 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/a6399a72822a48de8f3cffcf4c375125, entries=250, sequenceid=222, filesize=16.5 K 2024-12-09T01:43:04,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/b93de55e55fe44b88ba13d280b006bd3 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/b93de55e55fe44b88ba13d280b006bd3 2024-12-09T01:43:04,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/b93de55e55fe44b88ba13d280b006bd3, entries=150, sequenceid=222, filesize=11.9 K 2024-12-09T01:43:04,029 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/efdbbf24f55b42569ec758bd829db0d5 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/efdbbf24f55b42569ec758bd829db0d5 2024-12-09T01:43:04,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/efdbbf24f55b42569ec758bd829db0d5, entries=150, sequenceid=222, filesize=11.9 K 2024-12-09T01:43:04,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for f66467034fd235041292f78356d98c07 in 593ms, sequenceid=222, compaction requested=false 2024-12-09T01:43:04,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:04,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-12-09T01:43:04,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:04,074 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T01:43:04,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:04,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:04,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:04,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:04,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:04,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:04,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/f25c3dd308b7413a92df77da2da7d1e3 is 50, key is test_row_0/A:col10/1733708583548/Put/seqid=0 2024-12-09T01:43:04,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742343_1519 (size=9757) 2024-12-09T01:43:04,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:04,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:04,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:04,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708644214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:04,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:04,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708644214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:04,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,223 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:04,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:04,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708644217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:04,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,224 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:04,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:04,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708644221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:04,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:04,233 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:04,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708644222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,234 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:04,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:04,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708644333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:04,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:04,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:04,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708644343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:04,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708644343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,355 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:04,489 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/f25c3dd308b7413a92df77da2da7d1e3 2024-12-09T01:43:04,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/fb31fe3ca3a84367a09d72830cbccf7a is 50, key is test_row_0/B:col10/1733708583548/Put/seqid=0 2024-12-09T01:43:04,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742344_1520 (size=9757) 2024-12-09T01:43:04,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:04,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708644553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:04,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:04,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:04,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708644563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:04,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:04,571 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:04,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708644563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:04,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,572 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:04,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:04,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708644734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:04,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:04,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708644734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:04,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,742 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:04,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:04,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:04,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708644863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:04,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,871 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:04,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:04,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708644884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:04,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708644884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:04,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:04,902 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/fb31fe3ca3a84367a09d72830cbccf7a 2024-12-09T01:43:04,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/0ceaf987aa6241b986f2938c1fce6472 is 50, key is test_row_0/C:col10/1733708583548/Put/seqid=0 2024-12-09T01:43:04,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742345_1521 (size=9757) 2024-12-09T01:43:04,913 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/0ceaf987aa6241b986f2938c1fce6472 2024-12-09T01:43:04,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/f25c3dd308b7413a92df77da2da7d1e3 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/f25c3dd308b7413a92df77da2da7d1e3 2024-12-09T01:43:04,921 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/f25c3dd308b7413a92df77da2da7d1e3, entries=100, sequenceid=245, filesize=9.5 K 2024-12-09T01:43:04,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/fb31fe3ca3a84367a09d72830cbccf7a as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/fb31fe3ca3a84367a09d72830cbccf7a 2024-12-09T01:43:04,924 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/fb31fe3ca3a84367a09d72830cbccf7a, entries=100, sequenceid=245, filesize=9.5 K 2024-12-09T01:43:04,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/0ceaf987aa6241b986f2938c1fce6472 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/0ceaf987aa6241b986f2938c1fce6472 2024-12-09T01:43:04,928 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/0ceaf987aa6241b986f2938c1fce6472, entries=100, sequenceid=245, filesize=9.5 K 2024-12-09T01:43:04,929 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f66467034fd235041292f78356d98c07 in 855ms, sequenceid=245, compaction requested=true 2024-12-09T01:43:04,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:04,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:04,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-09T01:43:04,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-09T01:43:04,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-09T01:43:04,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0930 sec 2024-12-09T01:43:04,934 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.0990 sec 2024-12-09T01:43:04,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-09T01:43:04,994 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:04,995 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:43:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-12-09T01:43:04,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T01:43:04,997 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:43:04,997 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:43:04,997 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:43:05,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T01:43:05,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-12-09T01:43:05,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:05,150 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T01:43:05,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:05,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:05,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:05,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:05,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:05,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:05,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/9cee9ad3224644e28f3367a6d4a2d956 is 50, key is test_row_0/A:col10/1733708584219/Put/seqid=0 2024-12-09T01:43:05,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742346_1522 (size=12151) 2024-12-09T01:43:05,158 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/9cee9ad3224644e28f3367a6d4a2d956 2024-12-09T01:43:05,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/df5773d5fe8c48429361f144c95b9b37 is 50, key is test_row_0/B:col10/1733708584219/Put/seqid=0 2024-12-09T01:43:05,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742347_1523 (size=12151) 2024-12-09T01:43:05,168 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/df5773d5fe8c48429361f144c95b9b37 2024-12-09T01:43:05,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/216d4e02d1764589bc00e7bae6ffcf13 is 50, key is test_row_0/C:col10/1733708584219/Put/seqid=0 2024-12-09T01:43:05,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742348_1524 (size=12151) 2024-12-09T01:43:05,178 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/216d4e02d1764589bc00e7bae6ffcf13 2024-12-09T01:43:05,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/9cee9ad3224644e28f3367a6d4a2d956 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/9cee9ad3224644e28f3367a6d4a2d956 2024-12-09T01:43:05,187 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/9cee9ad3224644e28f3367a6d4a2d956, entries=150, sequenceid=257, filesize=11.9 K 2024-12-09T01:43:05,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/df5773d5fe8c48429361f144c95b9b37 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/df5773d5fe8c48429361f144c95b9b37 2024-12-09T01:43:05,194 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/df5773d5fe8c48429361f144c95b9b37, entries=150, sequenceid=257, filesize=11.9 K 2024-12-09T01:43:05,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/216d4e02d1764589bc00e7bae6ffcf13 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/216d4e02d1764589bc00e7bae6ffcf13 2024-12-09T01:43:05,202 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/216d4e02d1764589bc00e7bae6ffcf13, entries=150, sequenceid=257, filesize=11.9 K 2024-12-09T01:43:05,203 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=0 B/0 for f66467034fd235041292f78356d98c07 in 53ms, sequenceid=257, compaction requested=true 2024-12-09T01:43:05,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:05,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:05,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-09T01:43:05,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-12-09T01:43:05,208 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-12-09T01:43:05,208 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 209 msec 2024-12-09T01:43:05,210 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 213 msec 2024-12-09T01:43:05,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-09T01:43:05,314 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:05,315 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:43:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-12-09T01:43:05,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T01:43:05,322 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:43:05,323 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:43:05,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:43:05,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:05,409 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:43:05,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:05,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:05,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:05,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:05,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:05,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:05,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/6f0282b59deb4f1fa80b78342c0a5c10 is 50, key is test_row_1/A:col10/1733708585408/Put/seqid=0 2024-12-09T01:43:05,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742349_1525 (size=12297) 2024-12-09T01:43:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T01:43:05,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/6f0282b59deb4f1fa80b78342c0a5c10 2024-12-09T01:43:05,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/9941a9b380e6431aad19e2528c39281f is 50, key is test_row_1/B:col10/1733708585408/Put/seqid=0 2024-12-09T01:43:05,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742350_1526 (size=9857) 2024-12-09T01:43:05,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T01:43:05,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:05,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:05,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:05,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:05,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:05,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:05,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733708585452, max timestamp=1733708585481Num uniques 859; 2024-12-09T01:43:05,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733708585452, max timestamp=1733708585481Num uniques 859; 2024-12-09T01:43:05,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T01:43:05,484 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T01:43:05,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708585454, max timestamp=1733708585481Num uniques 859; 2024-12-09T01:43:05,487 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:43:05,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:05,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708645543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:05,559 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:05,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708645554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:05,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:05,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:05,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:05,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708645559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:05,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T01:43:05,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:05,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:05,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:05,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:05,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:05,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:05,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T01:43:05,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:05,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708645663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:05,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:05,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:05,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:05,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:05,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708645674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:05,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:05,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:05,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708645683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:05,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708645754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:05,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:05,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708645754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:05,766 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_0' in f66467034fd235041292f78356d98c07 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:43:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:05,766 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_0' in f66467034fd235041292f78356d98c07 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:43:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:05,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T01:43:05,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:05,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:05,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:05,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:05,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:05,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:05,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/9941a9b380e6431aad19e2528c39281f 2024-12-09T01:43:05,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/2dbd43b916cb481bb5da66730d069276 is 50, key is test_row_1/C:col10/1733708585408/Put/seqid=0 2024-12-09T01:43:05,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742351_1527 (size=9857) 2024-12-09T01:43:05,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/2dbd43b916cb481bb5da66730d069276 2024-12-09T01:43:05,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/6f0282b59deb4f1fa80b78342c0a5c10 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/6f0282b59deb4f1fa80b78342c0a5c10 2024-12-09T01:43:05,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/6f0282b59deb4f1fa80b78342c0a5c10, entries=150, sequenceid=267, filesize=12.0 K 2024-12-09T01:43:05,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/9941a9b380e6431aad19e2528c39281f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/9941a9b380e6431aad19e2528c39281f 2024-12-09T01:43:05,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/9941a9b380e6431aad19e2528c39281f, entries=100, sequenceid=267, filesize=9.6 K 2024-12-09T01:43:05,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/2dbd43b916cb481bb5da66730d069276 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/2dbd43b916cb481bb5da66730d069276 2024-12-09T01:43:05,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/2dbd43b916cb481bb5da66730d069276, entries=100, sequenceid=267, filesize=9.6 K 2024-12-09T01:43:05,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f66467034fd235041292f78356d98c07 in 463ms, sequenceid=267, compaction requested=true 2024-12-09T01:43:05,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:05,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:43:05,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:05,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:43:05,872 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T01:43:05,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:05,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:43:05,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:05,872 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T01:43:05,874 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 56409 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T01:43:05,874 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 63629 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T01:43:05,874 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/B is initiating minor compaction (all files) 2024-12-09T01:43:05,874 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/A is initiating minor compaction (all files) 2024-12-09T01:43:05,874 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/B in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:05,874 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/A in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:05,874 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/d916e9b2008b4e80aac8582512917d10, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/b93de55e55fe44b88ba13d280b006bd3, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/fb31fe3ca3a84367a09d72830cbccf7a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/df5773d5fe8c48429361f144c95b9b37, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/9941a9b380e6431aad19e2528c39281f] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=55.1 K 2024-12-09T01:43:05,874 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/e826e9f3cb2f4006bf3048525fa5096f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/a6399a72822a48de8f3cffcf4c375125, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/f25c3dd308b7413a92df77da2da7d1e3, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/9cee9ad3224644e28f3367a6d4a2d956, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/6f0282b59deb4f1fa80b78342c0a5c10] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=62.1 K 2024-12-09T01:43:05,874 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting e826e9f3cb2f4006bf3048525fa5096f, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733708582094 2024-12-09T01:43:05,874 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting d916e9b2008b4e80aac8582512917d10, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733708582094 2024-12-09T01:43:05,875 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting b93de55e55fe44b88ba13d280b006bd3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733708583311 2024-12-09T01:43:05,875 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6399a72822a48de8f3cffcf4c375125, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733708583295 2024-12-09T01:43:05,875 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting fb31fe3ca3a84367a09d72830cbccf7a, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733708583544 2024-12-09T01:43:05,875 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting f25c3dd308b7413a92df77da2da7d1e3, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733708583544 2024-12-09T01:43:05,875 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting df5773d5fe8c48429361f144c95b9b37, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733708584215 2024-12-09T01:43:05,875 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cee9ad3224644e28f3367a6d4a2d956, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733708584215 2024-12-09T01:43:05,875 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 9941a9b380e6431aad19e2528c39281f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733708585406 2024-12-09T01:43:05,875 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f0282b59deb4f1fa80b78342c0a5c10, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733708585400 2024-12-09T01:43:05,888 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#B#compaction#453 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:05,888 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#A#compaction#452 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:05,889 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/5e3bf539c9fb4e519d3e3e6f57d6084e is 50, key is test_row_0/A:col10/1733708584219/Put/seqid=0 2024-12-09T01:43:05,889 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/3e3366ec799f4a71ab136a1ce364ef74 is 50, key is test_row_0/B:col10/1733708584219/Put/seqid=0 2024-12-09T01:43:05,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:05,896 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-09T01:43:05,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:05,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:05,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:05,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:05,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:05,896 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:05,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742352_1528 (size=12763) 2024-12-09T01:43:05,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742353_1529 (size=12763) 2024-12-09T01:43:05,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/4270d6c139a641d78f479507d1fde502 is 50, key is test_row_0/A:col10/1733708585557/Put/seqid=0 2024-12-09T01:43:05,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742354_1530 (size=14741) 2024-12-09T01:43:05,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/4270d6c139a641d78f479507d1fde502 2024-12-09T01:43:05,913 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/5e3bf539c9fb4e519d3e3e6f57d6084e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/5e3bf539c9fb4e519d3e3e6f57d6084e 2024-12-09T01:43:05,916 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f66467034fd235041292f78356d98c07/A of f66467034fd235041292f78356d98c07 into 5e3bf539c9fb4e519d3e3e6f57d6084e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:05,917 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:05,917 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/A, priority=11, startTime=1733708585872; duration=0sec 2024-12-09T01:43:05,917 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:05,917 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:A 2024-12-09T01:43:05,917 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T01:43:05,918 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 56409 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T01:43:05,918 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/C is initiating minor compaction (all files) 2024-12-09T01:43:05,918 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/C in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:05,918 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/c82ec52a3c2c4d3894f9995fe7582909, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/efdbbf24f55b42569ec758bd829db0d5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/0ceaf987aa6241b986f2938c1fce6472, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/216d4e02d1764589bc00e7bae6ffcf13, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/2dbd43b916cb481bb5da66730d069276] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=55.1 K 2024-12-09T01:43:05,919 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting c82ec52a3c2c4d3894f9995fe7582909, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1733708582094 2024-12-09T01:43:05,919 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting efdbbf24f55b42569ec758bd829db0d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=222, earliestPutTs=1733708583311 2024-12-09T01:43:05,919 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ceaf987aa6241b986f2938c1fce6472, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1733708583544 2024-12-09T01:43:05,919 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 216d4e02d1764589bc00e7bae6ffcf13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1733708584215 2024-12-09T01:43:05,920 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2dbd43b916cb481bb5da66730d069276, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733708585406 2024-12-09T01:43:05,921 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/51f5034fb63549debcad1ee84ed7aba4 is 50, key is test_row_0/B:col10/1733708585557/Put/seqid=0 2024-12-09T01:43:05,931 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#C#compaction#456 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:05,931 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/3348f7a5a3444642b192b468dde77bcb is 50, key is test_row_0/C:col10/1733708584219/Put/seqid=0 2024-12-09T01:43:05,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708645925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:05,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:05,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:05,932 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:05,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742355_1531 (size=12301) 2024-12-09T01:43:05,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/51f5034fb63549debcad1ee84ed7aba4 2024-12-09T01:43:05,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T01:43:05,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:05,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:05,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:05,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:05,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:05,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:05,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742356_1532 (size=12763) 2024-12-09T01:43:05,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T01:43:05,944 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/3348f7a5a3444642b192b468dde77bcb as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/3348f7a5a3444642b192b468dde77bcb 2024-12-09T01:43:05,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/ab8041e56c1542309341afdaad8f673d is 50, key is test_row_0/C:col10/1733708585557/Put/seqid=0 2024-12-09T01:43:05,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:05,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708645930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:05,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:05,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742357_1533 (size=12301) 2024-12-09T01:43:05,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/ab8041e56c1542309341afdaad8f673d 2024-12-09T01:43:05,951 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f66467034fd235041292f78356d98c07/C of f66467034fd235041292f78356d98c07 into 3348f7a5a3444642b192b468dde77bcb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:05,951 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:05,951 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/C, priority=11, startTime=1733708585872; duration=0sec 2024-12-09T01:43:05,951 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:05,951 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:C 2024-12-09T01:43:05,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:05,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708645932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:05,951 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:05,952 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:05,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/4270d6c139a641d78f479507d1fde502 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/4270d6c139a641d78f479507d1fde502 2024-12-09T01:43:05,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/4270d6c139a641d78f479507d1fde502, entries=200, sequenceid=303, filesize=14.4 K 2024-12-09T01:43:05,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/51f5034fb63549debcad1ee84ed7aba4 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/51f5034fb63549debcad1ee84ed7aba4 2024-12-09T01:43:05,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/51f5034fb63549debcad1ee84ed7aba4, entries=150, sequenceid=303, filesize=12.0 K 2024-12-09T01:43:05,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/ab8041e56c1542309341afdaad8f673d as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/ab8041e56c1542309341afdaad8f673d 2024-12-09T01:43:05,970 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/ab8041e56c1542309341afdaad8f673d, entries=150, sequenceid=303, filesize=12.0 K 2024-12-09T01:43:05,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for f66467034fd235041292f78356d98c07 in 75ms, sequenceid=303, compaction requested=false 2024-12-09T01:43:05,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:06,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:06,049 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:43:06,049 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:06,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:06,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:06,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:06,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:06,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:06,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/f2cfae5be68d4f60920979631283fc72 is 50, key is test_row_0/A:col10/1733708586048/Put/seqid=0 2024-12-09T01:43:06,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742358_1534 (size=14741) 2024-12-09T01:43:06,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/f2cfae5be68d4f60920979631283fc72 2024-12-09T01:43:06,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/6be43740ba8746bca55db2f3c2fdd7e8 is 50, key is test_row_0/B:col10/1733708586048/Put/seqid=0 2024-12-09T01:43:06,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742359_1535 (size=12301) 2024-12-09T01:43:06,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/6be43740ba8746bca55db2f3c2fdd7e8 2024-12-09T01:43:06,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T01:43:06,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:06,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:06,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:06,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:06,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/284e04123c0742f28823a7b6484b62e9 is 50, key is test_row_0/C:col10/1733708586048/Put/seqid=0 2024-12-09T01:43:06,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:06,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:06,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742360_1536 (size=12301) 2024-12-09T01:43:06,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/284e04123c0742f28823a7b6484b62e9 2024-12-09T01:43:06,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/f2cfae5be68d4f60920979631283fc72 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/f2cfae5be68d4f60920979631283fc72 2024-12-09T01:43:06,100 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/f2cfae5be68d4f60920979631283fc72, entries=200, sequenceid=315, filesize=14.4 K 2024-12-09T01:43:06,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/6be43740ba8746bca55db2f3c2fdd7e8 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/6be43740ba8746bca55db2f3c2fdd7e8 2024-12-09T01:43:06,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/6be43740ba8746bca55db2f3c2fdd7e8, entries=150, sequenceid=315, filesize=12.0 K 2024-12-09T01:43:06,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/284e04123c0742f28823a7b6484b62e9 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/284e04123c0742f28823a7b6484b62e9 2024-12-09T01:43:06,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/284e04123c0742f28823a7b6484b62e9, entries=150, sequenceid=315, filesize=12.0 K 2024-12-09T01:43:06,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=100.63 KB/103050 for f66467034fd235041292f78356d98c07 in 60ms, sequenceid=315, compaction requested=true 2024-12-09T01:43:06,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:06,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:A, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:43:06,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:06,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:43:06,110 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:06,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:06,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:43:06,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T01:43:06,111 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42245 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:06,111 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/A is initiating minor compaction (all files) 2024-12-09T01:43:06,111 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/A in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:06,111 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/5e3bf539c9fb4e519d3e3e6f57d6084e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/4270d6c139a641d78f479507d1fde502, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/f2cfae5be68d4f60920979631283fc72] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=41.3 K 2024-12-09T01:43:06,111 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e3bf539c9fb4e519d3e3e6f57d6084e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733708584219 2024-12-09T01:43:06,112 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4270d6c139a641d78f479507d1fde502, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733708585521 2024-12-09T01:43:06,112 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2cfae5be68d4f60920979631283fc72, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733708585923 2024-12-09T01:43:06,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:06,116 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T01:43:06,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:06,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:06,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:06,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:06,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:06,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:06,118 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#A#compaction#461 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:06,119 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/525c47638c3a4cdf81e3ccf570d92c31 is 50, key is test_row_0/A:col10/1733708586048/Put/seqid=0 2024-12-09T01:43:06,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/cfd742c58e8546da82519c19457df011 is 50, key is test_row_0/A:col10/1733708586101/Put/seqid=0 2024-12-09T01:43:06,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742361_1537 (size=12915) 2024-12-09T01:43:06,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742362_1538 (size=24505) 2024-12-09T01:43:06,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:06,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708646164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:06,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:06,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:06,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708646178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:06,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:06,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:06,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708646179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:06,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T01:43:06,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:06,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:06,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:06,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:06,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:06,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:06,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708646284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,294 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:06,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708646284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:06,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,298 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:06,308 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/3e3366ec799f4a71ab136a1ce364ef74 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/3e3366ec799f4a71ab136a1ce364ef74 2024-12-09T01:43:06,312 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in f66467034fd235041292f78356d98c07/B of f66467034fd235041292f78356d98c07 into 3e3366ec799f4a71ab136a1ce364ef74(size=12.5 K), total size for store is 36.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:06,312 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:06,312 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/B, priority=11, startTime=1733708585872; duration=0sec 2024-12-09T01:43:06,312 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T01:43:06,312 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:B 2024-12-09T01:43:06,312 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:B 2024-12-09T01:43:06,312 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:06,313 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:06,313 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/C is initiating minor compaction (all files) 2024-12-09T01:43:06,313 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/C in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:06,313 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/3348f7a5a3444642b192b468dde77bcb, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/ab8041e56c1542309341afdaad8f673d, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/284e04123c0742f28823a7b6484b62e9] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=36.5 K 2024-12-09T01:43:06,313 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 3348f7a5a3444642b192b468dde77bcb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733708584219 2024-12-09T01:43:06,314 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting ab8041e56c1542309341afdaad8f673d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733708585521 2024-12-09T01:43:06,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:06,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708646303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:06,314 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 284e04123c0742f28823a7b6484b62e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733708585929 2024-12-09T01:43:06,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,314 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:06,320 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#C#compaction#463 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:06,321 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/e379ba35f5e34f818bc15d8dc8df08c7 is 50, key is test_row_0/C:col10/1733708586048/Put/seqid=0 2024-12-09T01:43:06,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742363_1539 (size=12915) 2024-12-09T01:43:06,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T01:43:06,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:06,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:06,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:06,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:06,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:06,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:06,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T01:43:06,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708646503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:06,514 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:06,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708646503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:06,536 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/525c47638c3a4cdf81e3ccf570d92c31 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/525c47638c3a4cdf81e3ccf570d92c31 2024-12-09T01:43:06,537 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:06,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708646524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:06,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:06,537 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/cfd742c58e8546da82519c19457df011 2024-12-09T01:43:06,542 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f66467034fd235041292f78356d98c07/A of f66467034fd235041292f78356d98c07 into 525c47638c3a4cdf81e3ccf570d92c31(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:06,542 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:06,542 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/A, priority=13, startTime=1733708586110; duration=0sec 2024-12-09T01:43:06,542 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:06,543 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:A 2024-12-09T01:43:06,543 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:06,544 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:06,544 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/B is initiating minor compaction (all files) 2024-12-09T01:43:06,544 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/B in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:06,545 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/3e3366ec799f4a71ab136a1ce364ef74, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/51f5034fb63549debcad1ee84ed7aba4, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/6be43740ba8746bca55db2f3c2fdd7e8] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=36.5 K 2024-12-09T01:43:06,545 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3e3366ec799f4a71ab136a1ce364ef74, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1733708584219 2024-12-09T01:43:06,545 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51f5034fb63549debcad1ee84ed7aba4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1733708585521 2024-12-09T01:43:06,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/1bafe6a87b5d4ddda8e711e2d516f41b is 50, key is test_row_0/B:col10/1733708586101/Put/seqid=0 2024-12-09T01:43:06,547 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6be43740ba8746bca55db2f3c2fdd7e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733708585929 2024-12-09T01:43:06,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T01:43:06,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:06,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:06,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:06,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:06,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:06,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742364_1540 (size=12301) 2024-12-09T01:43:06,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/1bafe6a87b5d4ddda8e711e2d516f41b 2024-12-09T01:43:06,564 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#B#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:06,565 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/5acf3ae97ec04902956022a2075d6bd7 is 50, key is test_row_0/B:col10/1733708586048/Put/seqid=0 2024-12-09T01:43:06,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742365_1541 (size=12915) 2024-12-09T01:43:06,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/b946c991062949d48631f5ea88cc448f is 50, key is test_row_0/C:col10/1733708586101/Put/seqid=0 2024-12-09T01:43:06,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742366_1542 (size=12301) 2024-12-09T01:43:06,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T01:43:06,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:06,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:06,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:06,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:06,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:06,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:06,738 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/e379ba35f5e34f818bc15d8dc8df08c7 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/e379ba35f5e34f818bc15d8dc8df08c7 2024-12-09T01:43:06,742 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f66467034fd235041292f78356d98c07/C of f66467034fd235041292f78356d98c07 into e379ba35f5e34f818bc15d8dc8df08c7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:06,742 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:06,742 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/C, priority=13, startTime=1733708586110; duration=0sec 2024-12-09T01:43:06,742 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:06,742 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:C 2024-12-09T01:43:06,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:06,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708646823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:06,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708646823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,830 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:06,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:06,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708646844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:06,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T01:43:06,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:06,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:06,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:06,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:06,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:06,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:06,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:06,985 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/5acf3ae97ec04902956022a2075d6bd7 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/5acf3ae97ec04902956022a2075d6bd7 2024-12-09T01:43:06,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/b946c991062949d48631f5ea88cc448f 2024-12-09T01:43:06,993 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f66467034fd235041292f78356d98c07/B of f66467034fd235041292f78356d98c07 into 5acf3ae97ec04902956022a2075d6bd7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:06,993 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:06,993 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/B, priority=13, startTime=1733708586110; duration=0sec 2024-12-09T01:43:06,993 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:06,993 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:B 2024-12-09T01:43:06,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/cfd742c58e8546da82519c19457df011 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/cfd742c58e8546da82519c19457df011 2024-12-09T01:43:06,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/cfd742c58e8546da82519c19457df011, entries=400, sequenceid=335, filesize=23.9 K 2024-12-09T01:43:06,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/1bafe6a87b5d4ddda8e711e2d516f41b as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/1bafe6a87b5d4ddda8e711e2d516f41b 2024-12-09T01:43:07,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/1bafe6a87b5d4ddda8e711e2d516f41b, entries=150, sequenceid=335, filesize=12.0 K 2024-12-09T01:43:07,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/b946c991062949d48631f5ea88cc448f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/b946c991062949d48631f5ea88cc448f 2024-12-09T01:43:07,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/b946c991062949d48631f5ea88cc448f, entries=150, sequenceid=335, filesize=12.0 K 2024-12-09T01:43:07,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-12-09T01:43:07,007 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for f66467034fd235041292f78356d98c07 in 891ms, sequenceid=335, compaction requested=false 2024-12-09T01:43:07,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:07,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:07,007 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T01:43:07,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:07,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:07,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:07,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:07,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:07,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:07,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/c1a3eaaec8504ac6a9a636bcc3b81c4b is 50, key is test_row_0/A:col10/1733708586177/Put/seqid=0 2024-12-09T01:43:07,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742367_1543 (size=12301) 2024-12-09T01:43:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:07,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:07,427 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/c1a3eaaec8504ac6a9a636bcc3b81c4b 2024-12-09T01:43:07,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/8a6801a45e8c488c812f55e17446b635 is 50, key is test_row_0/B:col10/1733708586177/Put/seqid=0 2024-12-09T01:43:07,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742368_1544 (size=12301) 2024-12-09T01:43:07,441 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/8a6801a45e8c488c812f55e17446b635 2024-12-09T01:43:07,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/21291c9fd37e4219b134f4d0527b45ec is 50, key is test_row_0/C:col10/1733708586177/Put/seqid=0 2024-12-09T01:43:07,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708587391, max timestamp=1733708587447Num uniques 859; 2024-12-09T01:43:07,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708587391, max timestamp=1733708587447Num uniques 859; 2024-12-09T01:43:07,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:43:07,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:43:07,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708587391, max timestamp=1733708587447Num uniques 859; 2024-12-09T01:43:07,453 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:43:07,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742369_1545 (size=12301) 2024-12-09T01:43:07,459 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/21291c9fd37e4219b134f4d0527b45ec 2024-12-09T01:43:07,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/c1a3eaaec8504ac6a9a636bcc3b81c4b as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/c1a3eaaec8504ac6a9a636bcc3b81c4b 2024-12-09T01:43:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T01:43:07,470 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/c1a3eaaec8504ac6a9a636bcc3b81c4b, entries=150, sequenceid=351, filesize=12.0 K 2024-12-09T01:43:07,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/8a6801a45e8c488c812f55e17446b635 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/8a6801a45e8c488c812f55e17446b635 2024-12-09T01:43:07,475 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/8a6801a45e8c488c812f55e17446b635, entries=150, sequenceid=351, filesize=12.0 K 2024-12-09T01:43:07,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/21291c9fd37e4219b134f4d0527b45ec as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/21291c9fd37e4219b134f4d0527b45ec 2024-12-09T01:43:07,479 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/21291c9fd37e4219b134f4d0527b45ec, entries=150, sequenceid=351, filesize=12.0 K 2024-12-09T01:43:07,480 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=46.96 KB/48090 for f66467034fd235041292f78356d98c07 in 473ms, sequenceid=351, compaction requested=true 2024-12-09T01:43:07,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:07,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:07,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-12-09T01:43:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-12-09T01:43:07,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:07,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-12-09T01:43:07,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:07,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:07,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:07,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:07,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:07,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:07,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-12-09T01:43:07,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1590 sec 2024-12-09T01:43:07,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/057f29c9558f45c8ad1191caa0152ef1 is 50, key is test_row_0/A:col10/1733708587391/Put/seqid=0 2024-12-09T01:43:07,489 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.1730 sec 2024-12-09T01:43:07,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742370_1546 (size=12301) 2024-12-09T01:43:07,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/057f29c9558f45c8ad1191caa0152ef1 2024-12-09T01:43:07,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/db44a3696ff54d6e9dcb97f74128519b is 50, key is test_row_0/B:col10/1733708587391/Put/seqid=0 2024-12-09T01:43:07,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742371_1547 (size=12301) 2024-12-09T01:43:07,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/db44a3696ff54d6e9dcb97f74128519b 2024-12-09T01:43:07,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/b63f06cf14fd453b91be7d3cf845f1f4 is 50, key is test_row_0/C:col10/1733708587391/Put/seqid=0 2024-12-09T01:43:07,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742372_1548 (size=12301) 2024-12-09T01:43:07,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708587542, max timestamp=1733708587567Num uniques 859; 2024-12-09T01:43:07,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:43:07,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708587542, max timestamp=1733708587567Num uniques 859; 2024-12-09T01:43:07,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:43:07,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708587542, max timestamp=1733708587567Num uniques 859; 2024-12-09T01:43:07,571 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:43:07,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708647664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:07,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:07,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:07,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:07,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:07,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708647665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:07,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:07,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:07,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:07,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:07,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708647666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:07,687 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:07,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:07,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708647783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:07,798 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_0' in f66467034fd235041292f78356d98c07 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:43:07,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:07,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:07,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:07,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:07,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708647793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:07,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:07,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708647793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:07,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:07,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708647793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:07,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:07,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:07,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:07,802 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_0' in f66467034fd235041292f78356d98c07 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4254 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:43:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:07,803 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:07,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:07,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708647794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:07,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:07,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:07,810 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:07,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=376 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/b63f06cf14fd453b91be7d3cf845f1f4 2024-12-09T01:43:07,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/057f29c9558f45c8ad1191caa0152ef1 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/057f29c9558f45c8ad1191caa0152ef1 2024-12-09T01:43:07,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/057f29c9558f45c8ad1191caa0152ef1, entries=150, sequenceid=376, filesize=12.0 K 2024-12-09T01:43:07,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/db44a3696ff54d6e9dcb97f74128519b as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/db44a3696ff54d6e9dcb97f74128519b 2024-12-09T01:43:07,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/db44a3696ff54d6e9dcb97f74128519b, entries=150, sequenceid=376, filesize=12.0 K 2024-12-09T01:43:07,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/b63f06cf14fd453b91be7d3cf845f1f4 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/b63f06cf14fd453b91be7d3cf845f1f4 2024-12-09T01:43:07,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/b63f06cf14fd453b91be7d3cf845f1f4, entries=150, sequenceid=376, filesize=12.0 K 2024-12-09T01:43:07,950 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for f66467034fd235041292f78356d98c07 in 467ms, sequenceid=376, compaction requested=true 2024-12-09T01:43:07,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:07,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:43:07,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:07,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:43:07,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:07,950 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:43:07,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:43:07,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:07,950 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:43:07,951 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 62022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:43:07,951 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:43:07,951 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/A is initiating minor compaction (all files) 2024-12-09T01:43:07,951 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/B is initiating minor compaction (all files) 2024-12-09T01:43:07,951 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/A in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:07,951 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/B in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:07,951 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/525c47638c3a4cdf81e3ccf570d92c31, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/cfd742c58e8546da82519c19457df011, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/c1a3eaaec8504ac6a9a636bcc3b81c4b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/057f29c9558f45c8ad1191caa0152ef1] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=60.6 K 2024-12-09T01:43:07,951 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/5acf3ae97ec04902956022a2075d6bd7, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/1bafe6a87b5d4ddda8e711e2d516f41b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/8a6801a45e8c488c812f55e17446b635, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/db44a3696ff54d6e9dcb97f74128519b] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=48.7 K 2024-12-09T01:43:07,952 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 525c47638c3a4cdf81e3ccf570d92c31, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733708585929 2024-12-09T01:43:07,952 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 5acf3ae97ec04902956022a2075d6bd7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733708585929 2024-12-09T01:43:07,952 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfd742c58e8546da82519c19457df011, keycount=400, bloomtype=ROW, size=23.9 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733708586068 2024-12-09T01:43:07,952 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bafe6a87b5d4ddda8e711e2d516f41b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733708586082 2024-12-09T01:43:07,952 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1a3eaaec8504ac6a9a636bcc3b81c4b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733708586162 2024-12-09T01:43:07,952 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a6801a45e8c488c812f55e17446b635, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733708586162 2024-12-09T01:43:07,953 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 057f29c9558f45c8ad1191caa0152ef1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1733708587391 2024-12-09T01:43:07,953 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting db44a3696ff54d6e9dcb97f74128519b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1733708587391 2024-12-09T01:43:07,971 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#A#compaction#473 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:07,972 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/d9ac08f1c82e4c7e8745458135573beb is 50, key is test_row_0/A:col10/1733708587391/Put/seqid=0 2024-12-09T01:43:07,972 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#B#compaction#474 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:07,972 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/de4ca5600e6f47dcb7935ef723946225 is 50, key is test_row_0/B:col10/1733708587391/Put/seqid=0 2024-12-09T01:43:07,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742373_1549 (size=13051) 2024-12-09T01:43:07,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742374_1550 (size=13051) 2024-12-09T01:43:07,997 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/d9ac08f1c82e4c7e8745458135573beb as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/d9ac08f1c82e4c7e8745458135573beb 2024-12-09T01:43:08,000 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/de4ca5600e6f47dcb7935ef723946225 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/de4ca5600e6f47dcb7935ef723946225 2024-12-09T01:43:08,003 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f66467034fd235041292f78356d98c07/A of f66467034fd235041292f78356d98c07 into d9ac08f1c82e4c7e8745458135573beb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:08,003 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:08,003 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/A, priority=12, startTime=1733708587950; duration=0sec 2024-12-09T01:43:08,003 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:08,003 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:A 2024-12-09T01:43:08,004 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:43:08,006 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49818 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:43:08,006 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/C is initiating minor compaction (all files) 2024-12-09T01:43:08,006 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f66467034fd235041292f78356d98c07/B of f66467034fd235041292f78356d98c07 into de4ca5600e6f47dcb7935ef723946225(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:08,006 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/C in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:08,006 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:08,006 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/B, priority=12, startTime=1733708587950; duration=0sec 2024-12-09T01:43:08,006 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/e379ba35f5e34f818bc15d8dc8df08c7, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/b946c991062949d48631f5ea88cc448f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/21291c9fd37e4219b134f4d0527b45ec, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/b63f06cf14fd453b91be7d3cf845f1f4] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=48.7 K 2024-12-09T01:43:08,006 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:08,006 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:B 2024-12-09T01:43:08,008 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting e379ba35f5e34f818bc15d8dc8df08c7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1733708585929 2024-12-09T01:43:08,009 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting b946c991062949d48631f5ea88cc448f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1733708586082 2024-12-09T01:43:08,009 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21291c9fd37e4219b134f4d0527b45ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1733708586162 2024-12-09T01:43:08,009 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting b63f06cf14fd453b91be7d3cf845f1f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1733708587391 2024-12-09T01:43:08,020 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#C#compaction#475 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:08,021 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/bd361cef26eb4ad481907a664774510f is 50, key is test_row_0/C:col10/1733708587391/Put/seqid=0 2024-12-09T01:43:08,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:08,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-12-09T01:43:08,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:08,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T01:43:08,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:08,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T01:43:08,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:08,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T01:43:08,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/fcd6dfd8b0e042dab06483381198c0a1 is 50, key is test_row_0/A:col10/1733708588021/Put/seqid=0 2024-12-09T01:43:08,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742375_1551 (size=13051) 2024-12-09T01:43:08,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742376_1552 (size=17181) 2024-12-09T01:43:08,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/fcd6dfd8b0e042dab06483381198c0a1 2024-12-09T01:43:08,045 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/bd361cef26eb4ad481907a664774510f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/bd361cef26eb4ad481907a664774510f 2024-12-09T01:43:08,050 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f66467034fd235041292f78356d98c07/C of f66467034fd235041292f78356d98c07 into bd361cef26eb4ad481907a664774510f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:08,050 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:08,051 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/C, priority=12, startTime=1733708587950; duration=0sec 2024-12-09T01:43:08,051 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:08,051 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:C 2024-12-09T01:43:08,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/882bb051af374c73aa7c2a8d03460795 is 50, key is test_row_0/B:col10/1733708588021/Put/seqid=0 2024-12-09T01:43:08,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742377_1553 (size=12301) 2024-12-09T01:43:08,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:08,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708648046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:08,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708648047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:08,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:08,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708648055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,064 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:08,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:08,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708648163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:08,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:08,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708648163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:08,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:08,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:08,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708648173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:08,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:08,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:08,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708648383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:08,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708648383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:08,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708648383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:08,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/882bb051af374c73aa7c2a8d03460795 2024-12-09T01:43:08,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/06f1bfe5b885428688fa4aa9c44ac27a is 50, key is test_row_0/C:col10/1733708588021/Put/seqid=0 2024-12-09T01:43:08,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742378_1554 (size=12301) 2024-12-09T01:43:08,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:08,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708648703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:08,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708648703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:08,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708648703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:08,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:08,710 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:08,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/06f1bfe5b885428688fa4aa9c44ac27a 2024-12-09T01:43:08,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/fcd6dfd8b0e042dab06483381198c0a1 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/fcd6dfd8b0e042dab06483381198c0a1 2024-12-09T01:43:08,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/fcd6dfd8b0e042dab06483381198c0a1, entries=250, sequenceid=416, filesize=16.8 K 2024-12-09T01:43:08,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/882bb051af374c73aa7c2a8d03460795 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/882bb051af374c73aa7c2a8d03460795 2024-12-09T01:43:08,883 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/882bb051af374c73aa7c2a8d03460795, entries=150, sequenceid=416, filesize=12.0 K 2024-12-09T01:43:08,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/06f1bfe5b885428688fa4aa9c44ac27a as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/06f1bfe5b885428688fa4aa9c44ac27a 2024-12-09T01:43:08,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/06f1bfe5b885428688fa4aa9c44ac27a, entries=150, sequenceid=416, filesize=12.0 K 2024-12-09T01:43:08,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for f66467034fd235041292f78356d98c07 in 862ms, sequenceid=416, compaction requested=false 2024-12-09T01:43:08,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:09,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:09,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:43:09,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:09,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:09,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:09,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:09,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:09,220 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:09,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/95853d055dda447baba2803ec465982d is 50, key is test_row_0/A:col10/1733708589216/Put/seqid=0 2024-12-09T01:43:09,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742379_1555 (size=17181) 2024-12-09T01:43:09,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708589238, max timestamp=1733708589286Num uniques 859; 2024-12-09T01:43:09,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708589238, max timestamp=1733708589286Num uniques 859; 2024-12-09T01:43:09,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1733708589238, max timestamp=1733708589286Num uniques 859; 2024-12-09T01:43:09,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:43:09,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:43:09,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-12-09T01:43:09,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708649363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:09,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:09,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:09,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:09,381 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708649365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:09,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:09,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:09,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708649365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:09,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:09,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:09,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:09,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-12-09T01:43:09,474 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:09,474 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:43:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-12-09T01:43:09,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T01:43:09,476 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:43:09,477 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:43:09,477 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:43:09,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708649484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:09,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:09,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:09,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:09,496 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708649484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:09,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:09,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:09,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:09,496 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708649484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:09,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T01:43:09,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T01:43:09,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:09,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:09,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:09,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:09,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:09,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/95853d055dda447baba2803ec465982d 2024-12-09T01:43:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:09,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/5fe7d69a54a24d74b614ab6fa9b144d7 is 50, key is test_row_0/B:col10/1733708589216/Put/seqid=0 2024-12-09T01:43:09,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742380_1556 (size=12301) 2024-12-09T01:43:09,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/5fe7d69a54a24d74b614ab6fa9b144d7 2024-12-09T01:43:09,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/6e10f28c8bd84fa29c3b5c59a8ab849e is 50, key is test_row_0/C:col10/1733708589216/Put/seqid=0 2024-12-09T01:43:09,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742381_1557 (size=12301) 2024-12-09T01:43:09,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708649704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:09,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:09,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:09,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708649704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:09,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:09,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:09,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:09,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:09,713 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708649704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:09,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:09,713 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:09,714 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:09,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T01:43:09,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:09,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:09,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:09,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:09,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:09,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T01:43:09,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T01:43:09,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:09,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:09,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:09,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:09,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:09,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:10,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708650024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:10,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:10,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708650024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:10,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:10,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708650024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:10,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:10,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=428 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/6e10f28c8bd84fa29c3b5c59a8ab849e 2024-12-09T01:43:10,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/95853d055dda447baba2803ec465982d as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/95853d055dda447baba2803ec465982d 2024-12-09T01:43:10,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/95853d055dda447baba2803ec465982d, entries=250, sequenceid=428, filesize=16.8 K 2024-12-09T01:43:10,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/5fe7d69a54a24d74b614ab6fa9b144d7 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/5fe7d69a54a24d74b614ab6fa9b144d7 2024-12-09T01:43:10,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/5fe7d69a54a24d74b614ab6fa9b144d7, entries=150, sequenceid=428, filesize=12.0 K 2024-12-09T01:43:10,069 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/6e10f28c8bd84fa29c3b5c59a8ab849e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/6e10f28c8bd84fa29c3b5c59a8ab849e 2024-12-09T01:43:10,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/6e10f28c8bd84fa29c3b5c59a8ab849e, entries=150, sequenceid=428, filesize=12.0 K 2024-12-09T01:43:10,074 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for f66467034fd235041292f78356d98c07 in 855ms, sequenceid=428, compaction requested=true 2024-12-09T01:43:10,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:10,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:43:10,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:10,075 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:10,075 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:10,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:43:10,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:10,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:43:10,075 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:10,080 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:10,080 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:10,080 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/B is initiating minor compaction (all files) 2024-12-09T01:43:10,080 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/A is initiating minor compaction (all files) 2024-12-09T01:43:10,080 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/A in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:10,080 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/B in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:10,080 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/d9ac08f1c82e4c7e8745458135573beb, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/fcd6dfd8b0e042dab06483381198c0a1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/95853d055dda447baba2803ec465982d] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=46.3 K 2024-12-09T01:43:10,080 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/de4ca5600e6f47dcb7935ef723946225, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/882bb051af374c73aa7c2a8d03460795, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/5fe7d69a54a24d74b614ab6fa9b144d7] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=36.8 K 2024-12-09T01:43:10,081 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting d9ac08f1c82e4c7e8745458135573beb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1733708587391 2024-12-09T01:43:10,081 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting de4ca5600e6f47dcb7935ef723946225, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1733708587391 2024-12-09T01:43:10,081 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 882bb051af374c73aa7c2a8d03460795, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733708587662 2024-12-09T01:43:10,081 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcd6dfd8b0e042dab06483381198c0a1, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733708587662 2024-12-09T01:43:10,081 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fe7d69a54a24d74b614ab6fa9b144d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733708589216 2024-12-09T01:43:10,081 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95853d055dda447baba2803ec465982d, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733708588042 2024-12-09T01:43:10,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-12-09T01:43:10,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:10,089 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-12-09T01:43:10,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:10,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:10,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:10,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:10,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:10,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:10,094 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#B#compaction#482 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:10,094 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/672e8f6d29134995ab3791e71a2b28be is 50, key is test_row_0/B:col10/1733708589216/Put/seqid=0 2024-12-09T01:43:10,097 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#A#compaction#483 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:10,098 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/0af59de7040b4fc49995b8c866b9b50f is 50, key is test_row_0/A:col10/1733708589216/Put/seqid=0 2024-12-09T01:43:10,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/2434ff6460f34fd282f7d6153810e07e is 50, key is test_row_0/A:col10/1733708589362/Put/seqid=0 2024-12-09T01:43:10,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T01:43:10,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742383_1559 (size=12301) 2024-12-09T01:43:10,111 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/2434ff6460f34fd282f7d6153810e07e 2024-12-09T01:43:10,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742382_1558 (size=13153) 2024-12-09T01:43:10,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/0d22ad93afb4492ea0601a4aacad84ad is 50, key is test_row_0/B:col10/1733708589362/Put/seqid=0 2024-12-09T01:43:10,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742384_1560 (size=13153) 2024-12-09T01:43:10,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742385_1561 (size=12301) 2024-12-09T01:43:10,124 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/0d22ad93afb4492ea0601a4aacad84ad 2024-12-09T01:43:10,128 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/0af59de7040b4fc49995b8c866b9b50f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/0af59de7040b4fc49995b8c866b9b50f 2024-12-09T01:43:10,133 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f66467034fd235041292f78356d98c07/A of f66467034fd235041292f78356d98c07 into 0af59de7040b4fc49995b8c866b9b50f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:10,133 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:10,133 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/A, priority=13, startTime=1733708590075; duration=0sec 2024-12-09T01:43:10,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/6dfd4034a04a4093a4fe158bdc56c637 is 50, key is test_row_0/C:col10/1733708589362/Put/seqid=0 2024-12-09T01:43:10,133 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:10,133 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:A 2024-12-09T01:43:10,133 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:10,135 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:10,135 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/C is initiating minor compaction (all files) 2024-12-09T01:43:10,135 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/C in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:10,135 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/bd361cef26eb4ad481907a664774510f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/06f1bfe5b885428688fa4aa9c44ac27a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/6e10f28c8bd84fa29c3b5c59a8ab849e] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=36.8 K 2024-12-09T01:43:10,135 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd361cef26eb4ad481907a664774510f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=376, earliestPutTs=1733708587391 2024-12-09T01:43:10,135 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06f1bfe5b885428688fa4aa9c44ac27a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1733708587662 2024-12-09T01:43:10,136 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e10f28c8bd84fa29c3b5c59a8ab849e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733708589216 2024-12-09T01:43:10,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742386_1562 (size=12301) 2024-12-09T01:43:10,143 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#C#compaction#487 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:10,143 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/d1ef94dcb9034bc9b81508542a6f7e11 is 50, key is test_row_0/C:col10/1733708589216/Put/seqid=0 2024-12-09T01:43:10,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742387_1563 (size=13153) 2024-12-09T01:43:10,150 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/d1ef94dcb9034bc9b81508542a6f7e11 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/d1ef94dcb9034bc9b81508542a6f7e11 2024-12-09T01:43:10,154 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f66467034fd235041292f78356d98c07/C of f66467034fd235041292f78356d98c07 into d1ef94dcb9034bc9b81508542a6f7e11(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:10,154 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:10,154 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/C, priority=13, startTime=1733708590075; duration=0sec 2024-12-09T01:43:10,154 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:10,154 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:C 2024-12-09T01:43:10,518 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/672e8f6d29134995ab3791e71a2b28be as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/672e8f6d29134995ab3791e71a2b28be 2024-12-09T01:43:10,522 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f66467034fd235041292f78356d98c07/B of f66467034fd235041292f78356d98c07 into 672e8f6d29134995ab3791e71a2b28be(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:10,523 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:10,523 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/B, priority=13, startTime=1733708590075; duration=0sec 2024-12-09T01:43:10,523 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:10,523 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:B 2024-12-09T01:43:10,538 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=464 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/6dfd4034a04a4093a4fe158bdc56c637 2024-12-09T01:43:10,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/2434ff6460f34fd282f7d6153810e07e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/2434ff6460f34fd282f7d6153810e07e 2024-12-09T01:43:10,546 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/2434ff6460f34fd282f7d6153810e07e, entries=150, sequenceid=464, filesize=12.0 K 2024-12-09T01:43:10,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/0d22ad93afb4492ea0601a4aacad84ad as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/0d22ad93afb4492ea0601a4aacad84ad 2024-12-09T01:43:10,550 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/0d22ad93afb4492ea0601a4aacad84ad, entries=150, sequenceid=464, filesize=12.0 K 2024-12-09T01:43:10,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:10,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:10,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/6dfd4034a04a4093a4fe158bdc56c637 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/6dfd4034a04a4093a4fe158bdc56c637 2024-12-09T01:43:10,556 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/6dfd4034a04a4093a4fe158bdc56c637, entries=150, sequenceid=464, filesize=12.0 K 2024-12-09T01:43:10,557 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=26.84 KB/27480 for f66467034fd235041292f78356d98c07 in 468ms, sequenceid=464, compaction requested=false 2024-12-09T01:43:10,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:10,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:10,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-09T01:43:10,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-09T01:43:10,561 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-09T01:43:10,561 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0810 sec 2024-12-09T01:43:10,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:10,562 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:43:10,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:10,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:10,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:10,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:10,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:10,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:10,563 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.0880 sec 2024-12-09T01:43:10,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/97be3d823a7c4eaa9cd87e84bdaf786b is 50, key is test_row_0/A:col10/1733708590560/Put/seqid=0 2024-12-09T01:43:10,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742388_1564 (size=17181) 2024-12-09T01:43:10,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-09T01:43:10,613 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:10,614 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:43:10,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-12-09T01:43:10,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T01:43:10,615 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:43:10,617 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:43:10,617 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:43:10,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708590571, max timestamp=1733708590669Num uniques 859; 2024-12-09T01:43:10,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708590571, max timestamp=1733708590669Num uniques 859; 2024-12-09T01:43:10,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:43:10,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:43:10,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708590571, max timestamp=1733708590669Num uniques 859; 2024-12-09T01:43:10,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:43:10,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T01:43:10,737 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:10,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708650716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:10,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:10,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708650730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:10,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:10,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708650731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:10,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-09T01:43:10,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:10,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:10,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:10,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:10,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:10,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:10,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708650844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:10,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:10,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:10,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:10,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708650844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:10,854 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:10,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:10,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:10,854 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:10,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:10,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708650853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:10,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:10,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:10,860 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:10,922 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-09T01:43:10,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:10,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:10,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:10,922 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:10,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:10,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:10,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T01:43:10,970 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/97be3d823a7c4eaa9cd87e84bdaf786b 2024-12-09T01:43:10,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/f665f513f1254a4eb1ff2bfceacdb6d9 is 50, key is test_row_0/B:col10/1733708590560/Put/seqid=0 2024-12-09T01:43:10,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742389_1565 (size=12301) 2024-12-09T01:43:11,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-09T01:43:11,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708651063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:11,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:11,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:11,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708651063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:11,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:11,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:11,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:11,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:11,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708651063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:11,077 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:11,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:11,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-09T01:43:11,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:11,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:11,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:11,230 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:11,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:11,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T01:43:11,382 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/f665f513f1254a4eb1ff2bfceacdb6d9 2024-12-09T01:43:11,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-09T01:43:11,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:11,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:11,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:11,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:11,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:11,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:11,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708651384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:11,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:11,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:11,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:11,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708651384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:11,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:11,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708651384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:11,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:11,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:11,390 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:11,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/09f585dc1064480e81c25b3b90ca450f is 50, key is test_row_0/C:col10/1733708590560/Put/seqid=0 2024-12-09T01:43:11,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:11,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:11,392 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:11,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742390_1566 (size=12301) 2024-12-09T01:43:11,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-09T01:43:11,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:11,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:11,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:11,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:11,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:11,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:11,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-09T01:43:11,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:11,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:11,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:11,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:11,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:11,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:11,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T01:43:11,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/09f585dc1064480e81c25b3b90ca450f 2024-12-09T01:43:11,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/97be3d823a7c4eaa9cd87e84bdaf786b as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/97be3d823a7c4eaa9cd87e84bdaf786b 2024-12-09T01:43:11,803 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/97be3d823a7c4eaa9cd87e84bdaf786b, entries=250, sequenceid=477, filesize=16.8 K 2024-12-09T01:43:11,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/f665f513f1254a4eb1ff2bfceacdb6d9 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/f665f513f1254a4eb1ff2bfceacdb6d9 2024-12-09T01:43:11,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/f665f513f1254a4eb1ff2bfceacdb6d9, entries=150, sequenceid=477, filesize=12.0 K 2024-12-09T01:43:11,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/09f585dc1064480e81c25b3b90ca450f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/09f585dc1064480e81c25b3b90ca450f 2024-12-09T01:43:11,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/09f585dc1064480e81c25b3b90ca450f, entries=150, sequenceid=477, filesize=12.0 K 2024-12-09T01:43:11,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for f66467034fd235041292f78356d98c07 in 1251ms, sequenceid=477, compaction requested=true 2024-12-09T01:43:11,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:11,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:43:11,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:11,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:43:11,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:11,813 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:11,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:43:11,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T01:43:11,813 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:11,814 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:11,814 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/B is initiating minor compaction (all files) 2024-12-09T01:43:11,814 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/B in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:11,814 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/672e8f6d29134995ab3791e71a2b28be, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/0d22ad93afb4492ea0601a4aacad84ad, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/f665f513f1254a4eb1ff2bfceacdb6d9] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=36.9 K 2024-12-09T01:43:11,815 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42635 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:11,815 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/A is initiating minor compaction (all files) 2024-12-09T01:43:11,815 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/A in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:11,815 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/0af59de7040b4fc49995b8c866b9b50f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/2434ff6460f34fd282f7d6153810e07e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/97be3d823a7c4eaa9cd87e84bdaf786b] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=41.6 K 2024-12-09T01:43:11,815 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0af59de7040b4fc49995b8c866b9b50f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733708589216 2024-12-09T01:43:11,815 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 672e8f6d29134995ab3791e71a2b28be, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733708589216 2024-12-09T01:43:11,815 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d22ad93afb4492ea0601a4aacad84ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1733708589346 2024-12-09T01:43:11,815 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2434ff6460f34fd282f7d6153810e07e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1733708589346 2024-12-09T01:43:11,816 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting f665f513f1254a4eb1ff2bfceacdb6d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1733708590552 2024-12-09T01:43:11,816 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97be3d823a7c4eaa9cd87e84bdaf786b, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1733708590552 2024-12-09T01:43:11,823 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#B#compaction#492 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:11,824 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/bc545062b2b54bcfa5566a4445342853 is 50, key is test_row_0/B:col10/1733708590560/Put/seqid=0 2024-12-09T01:43:11,824 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#A#compaction#491 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:11,825 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/ed0894dc99df40208c2879381d01a9ef is 50, key is test_row_0/A:col10/1733708590560/Put/seqid=0 2024-12-09T01:43:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:11,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=140.89 KB heapSize=395.53 KB 2024-12-09T01:43:11,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:11,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:11,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:11,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:11,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:11,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:11,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/8dcfae2f4db547f392e727bf10cf844f is 50, key is test_row_0/A:col10/1733708591825/Put/seqid=0 2024-12-09T01:43:11,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-09T01:43:11,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:11,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:11,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:11,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:11,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:11,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:11,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742392_1568 (size=13255) 2024-12-09T01:43:11,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742391_1567 (size=13255) 2024-12-09T01:43:11,861 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/ed0894dc99df40208c2879381d01a9ef as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/ed0894dc99df40208c2879381d01a9ef 2024-12-09T01:43:11,862 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/bc545062b2b54bcfa5566a4445342853 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/bc545062b2b54bcfa5566a4445342853 2024-12-09T01:43:11,868 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f66467034fd235041292f78356d98c07/A of f66467034fd235041292f78356d98c07 into ed0894dc99df40208c2879381d01a9ef(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:11,868 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:11,868 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/A, priority=13, startTime=1733708591813; duration=0sec 2024-12-09T01:43:11,868 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:11,868 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:A 2024-12-09T01:43:11,868 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:11,869 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f66467034fd235041292f78356d98c07/B of f66467034fd235041292f78356d98c07 into bc545062b2b54bcfa5566a4445342853(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:11,869 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:11,869 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/B, priority=13, startTime=1733708591813; duration=0sec 2024-12-09T01:43:11,870 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:11,870 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:B 2024-12-09T01:43:11,870 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:11,870 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/C is initiating minor compaction (all files) 2024-12-09T01:43:11,870 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/C in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:11,870 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/d1ef94dcb9034bc9b81508542a6f7e11, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/6dfd4034a04a4093a4fe158bdc56c637, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/09f585dc1064480e81c25b3b90ca450f] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=36.9 K 2024-12-09T01:43:11,871 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1ef94dcb9034bc9b81508542a6f7e11, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=428, earliestPutTs=1733708589216 2024-12-09T01:43:11,871 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dfd4034a04a4093a4fe158bdc56c637, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=464, earliestPutTs=1733708589346 2024-12-09T01:43:11,871 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09f585dc1064480e81c25b3b90ca450f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1733708590552 2024-12-09T01:43:11,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742393_1569 (size=14741) 2024-12-09T01:43:11,874 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/8dcfae2f4db547f392e727bf10cf844f 2024-12-09T01:43:11,878 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#C#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:11,879 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/4fd713d72b394da7add201a95ad65296 is 50, key is test_row_0/C:col10/1733708590560/Put/seqid=0 2024-12-09T01:43:11,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/c37ae193769d43ab9fd771d9faeefeb7 is 50, key is test_row_0/B:col10/1733708591825/Put/seqid=0 2024-12-09T01:43:11,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:11,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708651870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:11,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:11,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742394_1570 (size=13255) 2024-12-09T01:43:11,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:11,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708651872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:11,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:11,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:11,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:11,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742395_1571 (size=12301) 2024-12-09T01:43:11,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:11,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708651893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:11,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:11,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:11,909 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:11,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708651904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:11,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:11,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:11,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:11,917 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708651904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:11,917 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:11,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:11,918 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:11,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-09T01:43:11,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:11,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:11,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:11,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:11,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:11,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:12,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708651994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:12,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708651994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:12,006 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:12,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-09T01:43:12,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:12,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:12,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:12,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:12,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:12,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708652213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:12,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:12,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:12,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:12,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708652214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:12,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:12,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:12,293 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/4fd713d72b394da7add201a95ad65296 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/4fd713d72b394da7add201a95ad65296 2024-12-09T01:43:12,295 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/c37ae193769d43ab9fd771d9faeefeb7 2024-12-09T01:43:12,298 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f66467034fd235041292f78356d98c07/C of f66467034fd235041292f78356d98c07 into 4fd713d72b394da7add201a95ad65296(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:12,298 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:12,298 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/C, priority=13, startTime=1733708591813; duration=0sec 2024-12-09T01:43:12,298 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:12,298 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:C 2024-12-09T01:43:12,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/f7cf284d1034435190f5799af1cb6cf5 is 50, key is test_row_0/C:col10/1733708591825/Put/seqid=0 2024-12-09T01:43:12,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-09T01:43:12,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:12,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:12,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:12,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:12,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:12,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:12,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742396_1572 (size=12301) 2024-12-09T01:43:12,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-09T01:43:12,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:12,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:12,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:12,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:12,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:12,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708652533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:12,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:12,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:12,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:12,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708652533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:12,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:12,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:12,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:12,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-09T01:43:12,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:12,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:12,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:12,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:12,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:12,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/f7cf284d1034435190f5799af1cb6cf5 2024-12-09T01:43:12,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/8dcfae2f4db547f392e727bf10cf844f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/8dcfae2f4db547f392e727bf10cf844f 2024-12-09T01:43:12,719 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/8dcfae2f4db547f392e727bf10cf844f, entries=200, sequenceid=513, filesize=14.4 K 2024-12-09T01:43:12,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/c37ae193769d43ab9fd771d9faeefeb7 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/c37ae193769d43ab9fd771d9faeefeb7 2024-12-09T01:43:12,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/c37ae193769d43ab9fd771d9faeefeb7, entries=150, sequenceid=513, filesize=12.0 K 2024-12-09T01:43:12,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/f7cf284d1034435190f5799af1cb6cf5 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/f7cf284d1034435190f5799af1cb6cf5 2024-12-09T01:43:12,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/f7cf284d1034435190f5799af1cb6cf5, entries=150, sequenceid=513, filesize=12.0 K 2024-12-09T01:43:12,737 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~395.48 KB/404976, currentSize=40.25 KB/41220 for f66467034fd235041292f78356d98c07 in 907ms, sequenceid=513, compaction requested=false 2024-12-09T01:43:12,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T01:43:12,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-12-09T01:43:12,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:12,764 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-12-09T01:43:12,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:12,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:12,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:12,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:12,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:12,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:12,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/81ea8a02df684a3da90832d159583fcb is 50, key is test_row_0/A:col10/1733708591863/Put/seqid=0 2024-12-09T01:43:12,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742397_1573 (size=12301) 2024-12-09T01:43:12,782 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/81ea8a02df684a3da90832d159583fcb 2024-12-09T01:43:12,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/bfebfb3563524f59a12c93ffc04ab8db is 50, key is test_row_0/B:col10/1733708591863/Put/seqid=0 2024-12-09T01:43:12,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742398_1574 (size=12301) 2024-12-09T01:43:12,811 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/bfebfb3563524f59a12c93ffc04ab8db 2024-12-09T01:43:12,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/af27f6fe5f7643448a03aa750b940059 is 50, key is test_row_0/C:col10/1733708591863/Put/seqid=0 2024-12-09T01:43:12,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742399_1575 (size=12301) 2024-12-09T01:43:12,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:12,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:13,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708592973, max timestamp=1733708592998Num uniques 859; 2024-12-09T01:43:13,001 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:43:13,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708592973, max timestamp=1733708592998Num uniques 859; 2024-12-09T01:43:13,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:43:13,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708592973, max timestamp=1733708592998Num uniques 859; 2024-12-09T01:43:13,002 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:43:13,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708653065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,082 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708653073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708653074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708653075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708653076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708653193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708653194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,204 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,205 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708653194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,205 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708653204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708653204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,246 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=525 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/af27f6fe5f7643448a03aa750b940059 2024-12-09T01:43:13,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/81ea8a02df684a3da90832d159583fcb as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/81ea8a02df684a3da90832d159583fcb 2024-12-09T01:43:13,254 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/81ea8a02df684a3da90832d159583fcb, entries=150, sequenceid=525, filesize=12.0 K 2024-12-09T01:43:13,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/bfebfb3563524f59a12c93ffc04ab8db as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/bfebfb3563524f59a12c93ffc04ab8db 2024-12-09T01:43:13,258 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/bfebfb3563524f59a12c93ffc04ab8db, entries=150, sequenceid=525, filesize=12.0 K 2024-12-09T01:43:13,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/af27f6fe5f7643448a03aa750b940059 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/af27f6fe5f7643448a03aa750b940059 2024-12-09T01:43:13,262 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/af27f6fe5f7643448a03aa750b940059, entries=150, sequenceid=525, filesize=12.0 K 2024-12-09T01:43:13,263 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for f66467034fd235041292f78356d98c07 in 499ms, sequenceid=525, compaction requested=true 2024-12-09T01:43:13,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:13,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:13,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-09T01:43:13,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-09T01:43:13,266 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-09T01:43:13,266 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6470 sec 2024-12-09T01:43:13,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.6530 sec 2024-12-09T01:43:13,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:13,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=147.60 KB heapSize=415.45 KB 2024-12-09T01:43:13,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:13,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T01:43:13,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:13,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T01:43:13,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:13,433 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-12-09T01:43:13,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/8ca1aabcb1964de3908b808ede2537d4 is 50, key is test_row_0/A:col10/1733708593425/Put/seqid=0 2024-12-09T01:43:13,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708653437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,460 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708653439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708653439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708653441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,462 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708653456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742400_1576 (size=12301) 2024-12-09T01:43:13,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708653563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708653563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,575 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,575 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708653563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,576 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,588 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708653574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708653583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,595 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708653784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708653784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708653784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708653793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,805 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:13,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708653803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:13,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:13,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:13,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/8ca1aabcb1964de3908b808ede2537d4 2024-12-09T01:43:13,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/6f99fc7fb93740baa00d590e14e62f38 is 50, key is test_row_0/B:col10/1733708593425/Put/seqid=0 2024-12-09T01:43:13,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742401_1577 (size=12301) 2024-12-09T01:43:13,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/6f99fc7fb93740baa00d590e14e62f38 2024-12-09T01:43:13,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/946945301b9e47f687e7997b7a072361 is 50, key is test_row_0/C:col10/1733708593425/Put/seqid=0 2024-12-09T01:43:13,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742402_1578 (size=12301) 2024-12-09T01:43:13,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/946945301b9e47f687e7997b7a072361 2024-12-09T01:43:13,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/8ca1aabcb1964de3908b808ede2537d4 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/8ca1aabcb1964de3908b808ede2537d4 2024-12-09T01:43:13,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/8ca1aabcb1964de3908b808ede2537d4, entries=150, sequenceid=563, filesize=12.0 K 2024-12-09T01:43:13,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/6f99fc7fb93740baa00d590e14e62f38 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/6f99fc7fb93740baa00d590e14e62f38 2024-12-09T01:43:13,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/6f99fc7fb93740baa00d590e14e62f38, entries=150, sequenceid=563, filesize=12.0 K 2024-12-09T01:43:13,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/946945301b9e47f687e7997b7a072361 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/946945301b9e47f687e7997b7a072361 2024-12-09T01:43:13,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/946945301b9e47f687e7997b7a072361, entries=150, sequenceid=563, filesize=12.0 K 2024-12-09T01:43:13,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~436.03 KB/446496, currentSize=26.84 KB/27480 for f66467034fd235041292f78356d98c07 in 528ms, sequenceid=563, compaction requested=true 2024-12-09T01:43:13,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:13,957 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:43:13,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:43:13,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:13,958 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:43:13,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:43:13,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:13,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:43:13,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:13,959 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:43:13,959 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/B is initiating minor compaction (all files) 2024-12-09T01:43:13,959 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/B in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:13,959 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/bc545062b2b54bcfa5566a4445342853, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/c37ae193769d43ab9fd771d9faeefeb7, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/bfebfb3563524f59a12c93ffc04ab8db, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/6f99fc7fb93740baa00d590e14e62f38] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=49.0 K 2024-12-09T01:43:13,960 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52598 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:43:13,960 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/A is initiating minor compaction (all files) 2024-12-09T01:43:13,960 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/A in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:13,960 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/ed0894dc99df40208c2879381d01a9ef, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/8dcfae2f4db547f392e727bf10cf844f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/81ea8a02df684a3da90832d159583fcb, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/8ca1aabcb1964de3908b808ede2537d4] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=51.4 K 2024-12-09T01:43:13,961 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting bc545062b2b54bcfa5566a4445342853, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1733708590552 2024-12-09T01:43:13,962 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed0894dc99df40208c2879381d01a9ef, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1733708590552 2024-12-09T01:43:13,962 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting c37ae193769d43ab9fd771d9faeefeb7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733708590714 2024-12-09T01:43:13,962 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dcfae2f4db547f392e727bf10cf844f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733708590714 2024-12-09T01:43:13,962 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting bfebfb3563524f59a12c93ffc04ab8db, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733708591836 2024-12-09T01:43:13,962 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81ea8a02df684a3da90832d159583fcb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733708591836 2024-12-09T01:43:13,962 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f99fc7fb93740baa00d590e14e62f38, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1733708593072 2024-12-09T01:43:13,963 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ca1aabcb1964de3908b808ede2537d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1733708593072 2024-12-09T01:43:13,980 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#A#compaction#503 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:13,980 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/bd53ce478f064ce295f2535f51f7e060 is 50, key is test_row_0/A:col10/1733708593425/Put/seqid=0 2024-12-09T01:43:13,992 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#B#compaction#504 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:13,993 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/c3c0c48d635644d9a37fb6ecccc64f22 is 50, key is test_row_0/B:col10/1733708593425/Put/seqid=0 2024-12-09T01:43:14,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742403_1579 (size=13391) 2024-12-09T01:43:14,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742404_1580 (size=13391) 2024-12-09T01:43:14,039 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/c3c0c48d635644d9a37fb6ecccc64f22 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/c3c0c48d635644d9a37fb6ecccc64f22 2024-12-09T01:43:14,046 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f66467034fd235041292f78356d98c07/B of f66467034fd235041292f78356d98c07 into c3c0c48d635644d9a37fb6ecccc64f22(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:14,047 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:14,047 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/B, priority=12, startTime=1733708593958; duration=0sec 2024-12-09T01:43:14,047 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:14,047 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:B 2024-12-09T01:43:14,047 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:43:14,048 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:43:14,048 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/C is initiating minor compaction (all files) 2024-12-09T01:43:14,048 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/C in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:14,048 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/4fd713d72b394da7add201a95ad65296, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/f7cf284d1034435190f5799af1cb6cf5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/af27f6fe5f7643448a03aa750b940059, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/946945301b9e47f687e7997b7a072361] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=49.0 K 2024-12-09T01:43:14,048 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fd713d72b394da7add201a95ad65296, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1733708590552 2024-12-09T01:43:14,048 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting f7cf284d1034435190f5799af1cb6cf5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1733708590714 2024-12-09T01:43:14,048 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting af27f6fe5f7643448a03aa750b940059, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=525, earliestPutTs=1733708591836 2024-12-09T01:43:14,049 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 946945301b9e47f687e7997b7a072361, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1733708593072 2024-12-09T01:43:14,059 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#C#compaction#505 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:14,059 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/9a00dc2265fb4e988168d68f9bf04ce8 is 50, key is test_row_0/C:col10/1733708593425/Put/seqid=0 2024-12-09T01:43:14,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742405_1581 (size=13391) 2024-12-09T01:43:14,092 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/9a00dc2265fb4e988168d68f9bf04ce8 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/9a00dc2265fb4e988168d68f9bf04ce8 2024-12-09T01:43:14,099 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f66467034fd235041292f78356d98c07/C of f66467034fd235041292f78356d98c07 into 9a00dc2265fb4e988168d68f9bf04ce8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:14,099 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:14,099 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/C, priority=12, startTime=1733708593958; duration=0sec 2024-12-09T01:43:14,099 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:14,099 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:C 2024-12-09T01:43:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:14,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:43:14,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:14,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:14,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:14,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:14,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:14,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:14,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/23be3af0ea844d4fa1d6629919b44175 is 50, key is test_row_0/A:col10/1733708593454/Put/seqid=0 2024-12-09T01:43:14,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742406_1582 (size=14741) 2024-12-09T01:43:14,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:14,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733708594136, max timestamp=1733708594155Num uniques 859; 2024-12-09T01:43:14,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T01:43:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708654153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:14,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733708594136, max timestamp=1733708594155Num uniques 859; 2024-12-09T01:43:14,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T01:43:14,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1733708594136, max timestamp=1733708594155Num uniques 859; 2024-12-09T01:43:14,160 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T01:43:14,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:14,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:14,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708654223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:14,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:14,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:14,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708654239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:14,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,245 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:14,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708654240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:14,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:14,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708654242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:14,258 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:14,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,258 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:14,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708654264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:14,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708654354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:14,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708654354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:14,362 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:14,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:14,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:14,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708654363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:14,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:14,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708654363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:14,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:14,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,376 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:14,406 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/bd53ce478f064ce295f2535f51f7e060 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/bd53ce478f064ce295f2535f51f7e060 2024-12-09T01:43:14,410 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f66467034fd235041292f78356d98c07/A of f66467034fd235041292f78356d98c07 into bd53ce478f064ce295f2535f51f7e060(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:14,411 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:14,411 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/A, priority=12, startTime=1733708593957; duration=0sec 2024-12-09T01:43:14,411 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:14,411 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:A 2024-12-09T01:43:14,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:14,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708654483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:14,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,496 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:14,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/23be3af0ea844d4fa1d6629919b44175 2024-12-09T01:43:14,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/c91e7c6cfcf946388c71ec58883e904b is 50, key is test_row_0/B:col10/1733708593454/Put/seqid=0 2024-12-09T01:43:14,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742407_1583 (size=12301) 2024-12-09T01:43:14,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:14,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708654573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:14,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:14,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,577 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:14,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708654573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:14,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,577 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:14,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:14,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708654583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:14,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:14,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708654584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:14,593 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,594 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:14,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-09T01:43:14,784 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:14,784 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:43:14,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-12-09T01:43:14,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T01:43:14,786 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:43:14,787 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:43:14,787 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:43:14,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:14,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708654804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T01:43:14,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708654883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:14,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:14,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,894 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708654883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:14,895 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:14,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,895 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:14,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708654903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:14,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:14,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708654903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:14,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:14,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:14,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:14,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/c91e7c6cfcf946388c71ec58883e904b 2024-12-09T01:43:14,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T01:43:14,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:14,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:14,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:14,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:14,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:14,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/34005f12b8474fd2bcd611dc33f3224c is 50, key is test_row_0/C:col10/1733708593454/Put/seqid=0 2024-12-09T01:43:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:14,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742408_1584 (size=12301) 2024-12-09T01:43:15,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T01:43:15,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:15,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:15,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:15,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:15,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:15,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T01:43:15,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T01:43:15,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:15,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:15,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:15,246 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:15,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:15,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:15,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708655323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:15,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:15,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/34005f12b8474fd2bcd611dc33f3224c 2024-12-09T01:43:15,349 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/23be3af0ea844d4fa1d6629919b44175 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/23be3af0ea844d4fa1d6629919b44175 2024-12-09T01:43:15,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/23be3af0ea844d4fa1d6629919b44175, entries=200, sequenceid=575, filesize=14.4 K 2024-12-09T01:43:15,353 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/c91e7c6cfcf946388c71ec58883e904b as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/c91e7c6cfcf946388c71ec58883e904b 2024-12-09T01:43:15,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/c91e7c6cfcf946388c71ec58883e904b, entries=150, sequenceid=575, filesize=12.0 K 2024-12-09T01:43:15,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/34005f12b8474fd2bcd611dc33f3224c as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/34005f12b8474fd2bcd611dc33f3224c 2024-12-09T01:43:15,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/34005f12b8474fd2bcd611dc33f3224c, entries=150, sequenceid=575, filesize=12.0 K 2024-12-09T01:43:15,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for f66467034fd235041292f78356d98c07 in 1253ms, sequenceid=575, compaction requested=false 2024-12-09T01:43:15,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:15,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-12-09T01:43:15,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:15,399 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-12-09T01:43:15,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:15,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:15,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:15,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:15,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:15,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:15,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/10487e663ba24699be669e299c54709e is 50, key is test_row_0/A:col10/1733708594239/Put/seqid=0 2024-12-09T01:43:15,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742409_1585 (size=12301) 2024-12-09T01:43:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:15,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:15,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T01:43:15,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708655427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,436 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:15,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708655428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,441 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:15,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708655435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:15,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708655436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:15,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708655544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:15,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708655544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,554 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:15,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708655553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:15,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,560 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:15,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708655563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,574 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:15,767 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:15,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708655763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:15,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:15,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:15,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708655763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:15,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:15,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708655763, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:15,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:15,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708655784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:15,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:15,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:15,810 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=611 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/10487e663ba24699be669e299c54709e 2024-12-09T01:43:15,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/97305f2811274de28d3c1cb0fb53fc8f is 50, key is test_row_0/B:col10/1733708594239/Put/seqid=0 2024-12-09T01:43:15,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742410_1586 (size=12301) 2024-12-09T01:43:15,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T01:43:16,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:16,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708656073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:16,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708656073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,084 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:16,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708656073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,085 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:16,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:16,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708656093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:16,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:16,226 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=611 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/97305f2811274de28d3c1cb0fb53fc8f 2024-12-09T01:43:16,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/f96778ee388b4062bedb15b44f1308f4 is 50, key is test_row_0/C:col10/1733708594239/Put/seqid=0 2024-12-09T01:43:16,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742411_1587 (size=12301) 2024-12-09T01:43:16,242 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=611 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/f96778ee388b4062bedb15b44f1308f4 2024-12-09T01:43:16,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/10487e663ba24699be669e299c54709e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/10487e663ba24699be669e299c54709e 2024-12-09T01:43:16,253 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/10487e663ba24699be669e299c54709e, entries=150, sequenceid=611, filesize=12.0 K 2024-12-09T01:43:16,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/97305f2811274de28d3c1cb0fb53fc8f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/97305f2811274de28d3c1cb0fb53fc8f 2024-12-09T01:43:16,257 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/97305f2811274de28d3c1cb0fb53fc8f, entries=150, sequenceid=611, filesize=12.0 K 2024-12-09T01:43:16,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/f96778ee388b4062bedb15b44f1308f4 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/f96778ee388b4062bedb15b44f1308f4 2024-12-09T01:43:16,264 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/f96778ee388b4062bedb15b44f1308f4, entries=150, sequenceid=611, filesize=12.0 K 2024-12-09T01:43:16,264 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=46.96 KB/48090 for f66467034fd235041292f78356d98c07 in 865ms, sequenceid=611, compaction requested=true 2024-12-09T01:43:16,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:16,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:16,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-09T01:43:16,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-09T01:43:16,267 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-09T01:43:16,267 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4790 sec 2024-12-09T01:43:16,268 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.4830 sec 2024-12-09T01:43:16,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:16,349 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-12-09T01:43:16,349 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:16,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:16,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:16,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:16,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:16,350 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:16,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/98c941355cf44e0794d656b44556f97f is 50, key is test_row_0/A:col10/1733708595434/Put/seqid=0 2024-12-09T01:43:16,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742412_1588 (size=12297) 2024-12-09T01:43:16,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=622 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/98c941355cf44e0794d656b44556f97f 2024-12-09T01:43:16,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/8cbaac29b4d745139312718add8e8dd0 is 50, key is test_row_0/B:col10/1733708595434/Put/seqid=0 2024-12-09T01:43:16,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742413_1589 (size=9857) 2024-12-09T01:43:16,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708596457, max timestamp=1733708596509Num uniques 859; 2024-12-09T01:43:16,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:43:16,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708596457, max timestamp=1733708596509Num uniques 859; 2024-12-09T01:43:16,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:43:16,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1733708596457, max timestamp=1733708596509Num uniques 859; 2024-12-09T01:43:16,513 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-12-09T01:43:16,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:16,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708656609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:16,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708656610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:16,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:16,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:16,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708656613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:16,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:16,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708656625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:16,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:16,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:16,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708656626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:16,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:16,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708656733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:16,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:16,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708656734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:16,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:16,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708656744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708656753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:16,756 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:16,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,757 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:16,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=622 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/8cbaac29b4d745139312718add8e8dd0 2024-12-09T01:43:16,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/2433268f5cf044ccbf044d75770026b7 is 50, key is test_row_0/C:col10/1733708595434/Put/seqid=0 2024-12-09T01:43:16,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742414_1590 (size=9857) 2024-12-09T01:43:16,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-09T01:43:16,934 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:16,934 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:43:16,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-12-09T01:43:16,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T01:43:16,936 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:43:16,937 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:43:16,937 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:43:16,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:16,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708656944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:16,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:16,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708656944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:16,953 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:16,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:16,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708656963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:16,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:16,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708656963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:16,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:16,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:16,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T01:43:17,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-09T01:43:17,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:17,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:17,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:17,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:17,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:17,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:17,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=622 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/2433268f5cf044ccbf044d75770026b7 2024-12-09T01:43:17,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/98c941355cf44e0794d656b44556f97f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/98c941355cf44e0794d656b44556f97f 2024-12-09T01:43:17,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/98c941355cf44e0794d656b44556f97f, entries=150, sequenceid=622, filesize=12.0 K 2024-12-09T01:43:17,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/8cbaac29b4d745139312718add8e8dd0 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/8cbaac29b4d745139312718add8e8dd0 2024-12-09T01:43:17,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/8cbaac29b4d745139312718add8e8dd0, entries=100, sequenceid=622, filesize=9.6 K 2024-12-09T01:43:17,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/2433268f5cf044ccbf044d75770026b7 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/2433268f5cf044ccbf044d75770026b7 2024-12-09T01:43:17,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/2433268f5cf044ccbf044d75770026b7, entries=100, sequenceid=622, filesize=9.6 K 2024-12-09T01:43:17,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for f66467034fd235041292f78356d98c07 in 868ms, sequenceid=622, compaction requested=true 2024-12-09T01:43:17,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:17,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:43:17,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:17,218 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:43:17,218 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:43:17,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:43:17,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:17,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:43:17,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:17,219 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52730 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:43:17,219 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47850 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:43:17,219 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/A is initiating minor compaction (all files) 2024-12-09T01:43:17,219 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/B is initiating minor compaction (all files) 2024-12-09T01:43:17,219 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/A in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:17,219 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/B in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:17,219 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/bd53ce478f064ce295f2535f51f7e060, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/23be3af0ea844d4fa1d6629919b44175, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/10487e663ba24699be669e299c54709e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/98c941355cf44e0794d656b44556f97f] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=51.5 K 2024-12-09T01:43:17,219 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/c3c0c48d635644d9a37fb6ecccc64f22, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/c91e7c6cfcf946388c71ec58883e904b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/97305f2811274de28d3c1cb0fb53fc8f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/8cbaac29b4d745139312718add8e8dd0] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=46.7 K 2024-12-09T01:43:17,220 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd53ce478f064ce295f2535f51f7e060, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1733708593072 2024-12-09T01:43:17,220 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting c3c0c48d635644d9a37fb6ecccc64f22, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1733708593072 2024-12-09T01:43:17,220 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23be3af0ea844d4fa1d6629919b44175, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=575, earliestPutTs=1733708593454 2024-12-09T01:43:17,220 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting c91e7c6cfcf946388c71ec58883e904b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=575, earliestPutTs=1733708593454 2024-12-09T01:43:17,220 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10487e663ba24699be669e299c54709e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=611, earliestPutTs=1733708594217 2024-12-09T01:43:17,221 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 97305f2811274de28d3c1cb0fb53fc8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=611, earliestPutTs=1733708594217 2024-12-09T01:43:17,221 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98c941355cf44e0794d656b44556f97f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=622, earliestPutTs=1733708595426 2024-12-09T01:43:17,221 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cbaac29b4d745139312718add8e8dd0, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=622, earliestPutTs=1733708595434 2024-12-09T01:43:17,231 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#A#compaction#515 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:17,231 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#B#compaction#516 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:17,231 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/45bff516de3b4cdd8313cf88aefda6c7 is 50, key is test_row_0/A:col10/1733708595434/Put/seqid=0 2024-12-09T01:43:17,234 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/5d6807a7a54c4deda091dca76492ec4a is 50, key is test_row_0/B:col10/1733708595434/Put/seqid=0 2024-12-09T01:43:17,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742415_1591 (size=13527) 2024-12-09T01:43:17,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-12-09T01:43:17,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:17,243 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-12-09T01:43:17,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:17,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:17,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:17,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:17,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:17,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:17,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742416_1592 (size=13527) 2024-12-09T01:43:17,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/d6a288cc96f64db6b3503f8a707b84b7 is 50, key is test_row_0/A:col10/1733708596624/Put/seqid=0 2024-12-09T01:43:17,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T01:43:17,254 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/5d6807a7a54c4deda091dca76492ec4a as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/5d6807a7a54c4deda091dca76492ec4a 2024-12-09T01:43:17,258 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f66467034fd235041292f78356d98c07/B of f66467034fd235041292f78356d98c07 into 5d6807a7a54c4deda091dca76492ec4a(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:17,258 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:17,258 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/B, priority=12, startTime=1733708597218; duration=0sec 2024-12-09T01:43:17,258 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:17,258 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:B 2024-12-09T01:43:17,259 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-09T01:43:17,259 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47850 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-12-09T01:43:17,259 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/C is initiating minor compaction (all files) 2024-12-09T01:43:17,260 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/C in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:17,260 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/9a00dc2265fb4e988168d68f9bf04ce8, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/34005f12b8474fd2bcd611dc33f3224c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/f96778ee388b4062bedb15b44f1308f4, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/2433268f5cf044ccbf044d75770026b7] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=46.7 K 2024-12-09T01:43:17,260 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a00dc2265fb4e988168d68f9bf04ce8, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1733708593072 2024-12-09T01:43:17,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742417_1593 (size=12301) 2024-12-09T01:43:17,263 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 34005f12b8474fd2bcd611dc33f3224c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=575, earliestPutTs=1733708593454 2024-12-09T01:43:17,264 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting f96778ee388b4062bedb15b44f1308f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=611, earliestPutTs=1733708594217 2024-12-09T01:43:17,264 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/d6a288cc96f64db6b3503f8a707b84b7 2024-12-09T01:43:17,264 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 2433268f5cf044ccbf044d75770026b7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=622, earliestPutTs=1733708595434 2024-12-09T01:43:17,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/28269c4062d346669f85cb23145b8999 is 50, key is test_row_0/B:col10/1733708596624/Put/seqid=0 2024-12-09T01:43:17,272 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#C#compaction#519 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:17,273 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/88b58205d8eb4f72a0da628d10c0968f is 50, key is test_row_0/C:col10/1733708595434/Put/seqid=0 2024-12-09T01:43:17,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. as already flushing 2024-12-09T01:43:17,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742419_1595 (size=13527) 2024-12-09T01:43:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:17,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742418_1594 (size=12301) 2024-12-09T01:43:17,284 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/88b58205d8eb4f72a0da628d10c0968f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/88b58205d8eb4f72a0da628d10c0968f 2024-12-09T01:43:17,288 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f66467034fd235041292f78356d98c07/C of f66467034fd235041292f78356d98c07 into 88b58205d8eb4f72a0da628d10c0968f(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:17,288 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:17,288 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/C, priority=12, startTime=1733708597218; duration=0sec 2024-12-09T01:43:17,288 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:17,288 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:C 2024-12-09T01:43:17,310 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:17,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708657296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:17,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:17,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708657297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:17,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:17,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:17,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708657309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:17,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,318 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:17,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:17,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708657310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:17,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:17,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:17,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708657413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:17,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:17,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:17,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708657423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:17,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:17,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708657423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:17,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,433 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:17,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,433 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:17,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:17,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708657434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:17,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,437 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:17,540 INFO [Thread-2163 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2163. 2024-12-09T01:43:17,540 DEBUG [Thread-2163 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:43:17,540 DEBUG [Thread-2163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:17,540 DEBUG [Thread-2163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:17,540 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:43:17,544 INFO [Thread-2157 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2157. 2024-12-09T01:43:17,544 DEBUG [Thread-2157 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:43:17,544 DEBUG [Thread-2157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:17,544 DEBUG [Thread-2157 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:17,544 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:43:17,544 INFO [Thread-2159 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2159. 2024-12-09T01:43:17,544 DEBUG [Thread-2159 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:43:17,545 DEBUG [Thread-2159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:17,545 DEBUG [Thread-2159 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:17,545 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:43:17,552 INFO [Thread-2161 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2161. 2024-12-09T01:43:17,552 DEBUG [Thread-2161 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:43:17,552 DEBUG [Thread-2161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:17,553 DEBUG [Thread-2161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:17,553 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:43:17,555 INFO [Thread-2165 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2165. 2024-12-09T01:43:17,555 DEBUG [Thread-2165 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:43:17,555 DEBUG [Thread-2165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:17,555 DEBUG [Thread-2165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:17,555 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:43:17,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T01:43:17,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:17,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708657623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:17,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:17,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:17,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43678 deadline: 1733708657634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:17,634 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_1' in f66467034fd235041292f78356d98c07 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:43:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:17,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:17,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:17,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708657644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708657644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:17,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708657644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:17,645 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/45bff516de3b4cdd8313cf88aefda6c7 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/45bff516de3b4cdd8313cf88aefda6c7 2024-12-09T01:43:17,648 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f66467034fd235041292f78356d98c07/A of f66467034fd235041292f78356d98c07 into 45bff516de3b4cdd8313cf88aefda6c7(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:17,648 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:17,648 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/A, priority=12, startTime=1733708597217; duration=0sec 2024-12-09T01:43:17,648 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:17,648 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:A 2024-12-09T01:43:17,678 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/28269c4062d346669f85cb23145b8999 2024-12-09T01:43:17,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/9304149a1b60435f8628c71047255a41 is 50, key is test_row_0/C:col10/1733708596624/Put/seqid=0 2024-12-09T01:43:17,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742420_1596 (size=12301) 2024-12-09T01:43:17,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:17,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43596 deadline: 1733708657933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:17,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:17,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:17,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:17,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43690 deadline: 1733708657953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43662 deadline: 1733708657954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:17,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43634 deadline: 1733708657953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:17,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 , the old value is region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,954 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:17,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:17,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:17,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=2 because the exception is null or not the one we care about 2024-12-09T01:43:18,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T01:43:18,086 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=658 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/9304149a1b60435f8628c71047255a41 2024-12-09T01:43:18,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/d6a288cc96f64db6b3503f8a707b84b7 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/d6a288cc96f64db6b3503f8a707b84b7 2024-12-09T01:43:18,092 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/d6a288cc96f64db6b3503f8a707b84b7, entries=150, sequenceid=658, filesize=12.0 K 2024-12-09T01:43:18,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/28269c4062d346669f85cb23145b8999 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/28269c4062d346669f85cb23145b8999 2024-12-09T01:43:18,096 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/28269c4062d346669f85cb23145b8999, entries=150, sequenceid=658, filesize=12.0 K 2024-12-09T01:43:18,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/9304149a1b60435f8628c71047255a41 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/9304149a1b60435f8628c71047255a41 2024-12-09T01:43:18,099 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/9304149a1b60435f8628c71047255a41, entries=150, sequenceid=658, filesize=12.0 K 2024-12-09T01:43:18,099 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for f66467034fd235041292f78356d98c07 in 856ms, sequenceid=658, compaction requested=false 2024-12-09T01:43:18,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:18,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:18,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-12-09T01:43:18,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-12-09T01:43:18,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-09T01:43:18,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1640 sec 2024-12-09T01:43:18,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.1680 sec 2024-12-09T01:43:18,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on f66467034fd235041292f78356d98c07 2024-12-09T01:43:18,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T01:43:18,445 INFO [Thread-2148 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2148. 2024-12-09T01:43:18,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:18,445 DEBUG [Thread-2148 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:43:18,445 DEBUG [Thread-2148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:18,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:18,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:18,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:18,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:18,445 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:18,445 DEBUG [Thread-2148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:18,445 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:43:18,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/f2eb1e7aa4c54422b3bd6f4bc9d69522 is 50, key is test_row_0/A:col10/1733708597308/Put/seqid=0 2024-12-09T01:43:18,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742421_1597 (size=12301) 2024-12-09T01:43:18,466 INFO [Thread-2154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2154. 2024-12-09T01:43:18,466 INFO [Thread-2146 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2146. 2024-12-09T01:43:18,466 INFO [Thread-2152 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2152. 2024-12-09T01:43:18,466 DEBUG [Thread-2154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:43:18,466 DEBUG [Thread-2146 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:43:18,466 DEBUG [Thread-2152 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:43:18,466 DEBUG [Thread-2154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:18,466 DEBUG [Thread-2146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:18,466 DEBUG [Thread-2152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:18,466 DEBUG [Thread-2152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:18,466 DEBUG [Thread-2146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:18,466 DEBUG [Thread-2154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:18,466 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:43:18,466 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:43:18,466 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:43:18,852 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=673 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/f2eb1e7aa4c54422b3bd6f4bc9d69522 2024-12-09T01:43:18,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/b3b022387b904debb6be7b920afd7d35 is 50, key is test_row_0/B:col10/1733708597308/Put/seqid=0 2024-12-09T01:43:18,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742422_1598 (size=12301) 2024-12-09T01:43:19,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-09T01:43:19,084 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:19,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=673 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/b3b022387b904debb6be7b920afd7d35 2024-12-09T01:43:19,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/9070807c3d2c47b58e5ea8f74d864be0 is 50, key is test_row_0/C:col10/1733708597308/Put/seqid=0 2024-12-09T01:43:19,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742423_1599 (size=12301) 2024-12-09T01:43:19,655 INFO [Thread-2150 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2150. 2024-12-09T01:43:19,655 DEBUG [Thread-2150 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:43:19,655 DEBUG [Thread-2150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:19,656 DEBUG [Thread-2150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:19,656 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:43:19,656 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T01:43:19,656 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 140 2024-12-09T01:43:19,656 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-12-09T01:43:19,656 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 128 2024-12-09T01:43:19,656 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 141 2024-12-09T01:43:19,656 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-12-09T01:43:19,656 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T01:43:19,656 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T01:43:19,656 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 899 2024-12-09T01:43:19,656 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2693 rows 2024-12-09T01:43:19,656 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 907 2024-12-09T01:43:19,656 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2717 rows 2024-12-09T01:43:19,656 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 900 2024-12-09T01:43:19,656 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2698 rows 2024-12-09T01:43:19,656 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 901 2024-12-09T01:43:19,656 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2700 rows 2024-12-09T01:43:19,656 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 909 2024-12-09T01:43:19,656 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2722 rows 2024-12-09T01:43:19,656 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T01:43:19,657 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T01:43:19,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:19,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:19,657 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:43:19,658 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T01:43:19,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T01:43:19,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-09T01:43:19,660 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708599660"}]},"ts":"1733708599660"} 2024-12-09T01:43:19,661 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T01:43:19,662 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T01:43:19,662 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T01:43:19,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f66467034fd235041292f78356d98c07, UNASSIGN}] 2024-12-09T01:43:19,664 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f66467034fd235041292f78356d98c07, UNASSIGN 2024-12-09T01:43:19,665 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=f66467034fd235041292f78356d98c07, regionState=CLOSING, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:19,666 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f66467034fd235041292f78356d98c07, UNASSIGN because future has completed 2024-12-09T01:43:19,666 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T01:43:19,666 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943}] 2024-12-09T01:43:19,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=673 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/9070807c3d2c47b58e5ea8f74d864be0 2024-12-09T01:43:19,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/f2eb1e7aa4c54422b3bd6f4bc9d69522 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/f2eb1e7aa4c54422b3bd6f4bc9d69522 2024-12-09T01:43:19,676 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/f2eb1e7aa4c54422b3bd6f4bc9d69522, entries=150, sequenceid=673, filesize=12.0 K 2024-12-09T01:43:19,676 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/b3b022387b904debb6be7b920afd7d35 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/b3b022387b904debb6be7b920afd7d35 2024-12-09T01:43:19,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/b3b022387b904debb6be7b920afd7d35, entries=150, sequenceid=673, filesize=12.0 K 2024-12-09T01:43:19,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/9070807c3d2c47b58e5ea8f74d864be0 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/9070807c3d2c47b58e5ea8f74d864be0 2024-12-09T01:43:19,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/9070807c3d2c47b58e5ea8f74d864be0, entries=150, sequenceid=673, filesize=12.0 K 2024-12-09T01:43:19,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=26.84 KB/27480 for f66467034fd235041292f78356d98c07 in 1239ms, sequenceid=673, compaction requested=true 2024-12-09T01:43:19,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:19,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:43:19,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:19,683 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:19,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:43:19,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:19,683 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:19,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f66467034fd235041292f78356d98c07:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:43:19,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:19,684 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:19,684 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38129 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:19,684 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/B is initiating minor compaction (all files) 2024-12-09T01:43:19,684 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): f66467034fd235041292f78356d98c07/A is initiating minor compaction (all files) 2024-12-09T01:43:19,684 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/B in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:19,684 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f66467034fd235041292f78356d98c07/A in TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:19,684 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/5d6807a7a54c4deda091dca76492ec4a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/28269c4062d346669f85cb23145b8999, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/b3b022387b904debb6be7b920afd7d35] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=37.2 K 2024-12-09T01:43:19,684 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/45bff516de3b4cdd8313cf88aefda6c7, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/d6a288cc96f64db6b3503f8a707b84b7, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/f2eb1e7aa4c54422b3bd6f4bc9d69522] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp, totalSize=37.2 K 2024-12-09T01:43:19,684 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d6807a7a54c4deda091dca76492ec4a, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=622, earliestPutTs=1733708594240 2024-12-09T01:43:19,684 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 45bff516de3b4cdd8313cf88aefda6c7, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=622, earliestPutTs=1733708594240 2024-12-09T01:43:19,685 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 28269c4062d346669f85cb23145b8999, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1733708596581 2024-12-09T01:43:19,685 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6a288cc96f64db6b3503f8a707b84b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=658, earliestPutTs=1733708596581 2024-12-09T01:43:19,685 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting b3b022387b904debb6be7b920afd7d35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=673, earliestPutTs=1733708597294 2024-12-09T01:43:19,685 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2eb1e7aa4c54422b3bd6f4bc9d69522, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=673, earliestPutTs=1733708597294 2024-12-09T01:43:19,692 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#A#compaction#525 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:19,692 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/0cf4512711a64d998783e96070fa39c8 is 50, key is test_row_0/A:col10/1733708597308/Put/seqid=0 2024-12-09T01:43:19,692 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f66467034fd235041292f78356d98c07#B#compaction#524 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:19,693 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/1a22bb709af340679676da9a1bc81e91 is 50, key is test_row_0/B:col10/1733708597308/Put/seqid=0 2024-12-09T01:43:19,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742425_1601 (size=13629) 2024-12-09T01:43:19,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742424_1600 (size=13629) 2024-12-09T01:43:19,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-09T01:43:19,818 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close f66467034fd235041292f78356d98c07 2024-12-09T01:43:19,818 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T01:43:19,819 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing f66467034fd235041292f78356d98c07, disabling compactions & flushes 2024-12-09T01:43:19,819 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:19,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-09T01:43:20,102 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/1a22bb709af340679676da9a1bc81e91 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/1a22bb709af340679676da9a1bc81e91 2024-12-09T01:43:20,102 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/0cf4512711a64d998783e96070fa39c8 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/0cf4512711a64d998783e96070fa39c8 2024-12-09T01:43:20,106 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f66467034fd235041292f78356d98c07/A of f66467034fd235041292f78356d98c07 into 0cf4512711a64d998783e96070fa39c8(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:20,106 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f66467034fd235041292f78356d98c07/B of f66467034fd235041292f78356d98c07 into 1a22bb709af340679676da9a1bc81e91(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:20,106 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:20,106 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f66467034fd235041292f78356d98c07: 2024-12-09T01:43:20,106 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/A, priority=13, startTime=1733708599683; duration=0sec 2024-12-09T01:43:20,106 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., storeName=f66467034fd235041292f78356d98c07/B, priority=13, startTime=1733708599683; duration=0sec 2024-12-09T01:43:20,106 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:20,106 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:20,106 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:20,106 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:20,106 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:A 2024-12-09T01:43:20,106 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:B 2024-12-09T01:43:20,106 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. after waiting 0 ms 2024-12-09T01:43:20,106 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. because compaction request was cancelled 2024-12-09T01:43:20,106 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f66467034fd235041292f78356d98c07:C 2024-12-09T01:43:20,106 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:20,106 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(2902): Flushing f66467034fd235041292f78356d98c07 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-09T01:43:20,107 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=A 2024-12-09T01:43:20,107 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:20,107 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=B 2024-12-09T01:43:20,107 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:20,107 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK f66467034fd235041292f78356d98c07, store=C 2024-12-09T01:43:20,107 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:20,109 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/0a6dbb1ae0254e71a045dac0937b0d26 is 50, key is test_row_0/A:col10/1733708598464/Put/seqid=0 2024-12-09T01:43:20,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742426_1602 (size=9857) 2024-12-09T01:43:20,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-09T01:43:20,513 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/0a6dbb1ae0254e71a045dac0937b0d26 2024-12-09T01:43:20,519 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/cedb0dd0b7d44e259c17623eea14271f is 50, key is test_row_0/B:col10/1733708598464/Put/seqid=0 2024-12-09T01:43:20,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742427_1603 (size=9857) 2024-12-09T01:43:20,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-09T01:43:20,923 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/cedb0dd0b7d44e259c17623eea14271f 2024-12-09T01:43:20,928 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/9f18a7fccec6467d8b45df1e58dd73df is 50, key is test_row_0/C:col10/1733708598464/Put/seqid=0 2024-12-09T01:43:20,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742428_1604 (size=9857) 2024-12-09T01:43:21,331 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=682 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/9f18a7fccec6467d8b45df1e58dd73df 2024-12-09T01:43:21,335 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/A/0a6dbb1ae0254e71a045dac0937b0d26 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/0a6dbb1ae0254e71a045dac0937b0d26 2024-12-09T01:43:21,337 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/0a6dbb1ae0254e71a045dac0937b0d26, entries=100, sequenceid=682, filesize=9.6 K 2024-12-09T01:43:21,338 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/B/cedb0dd0b7d44e259c17623eea14271f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/cedb0dd0b7d44e259c17623eea14271f 2024-12-09T01:43:21,340 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/cedb0dd0b7d44e259c17623eea14271f, entries=100, sequenceid=682, filesize=9.6 K 2024-12-09T01:43:21,341 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/.tmp/C/9f18a7fccec6467d8b45df1e58dd73df as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/9f18a7fccec6467d8b45df1e58dd73df 2024-12-09T01:43:21,343 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/9f18a7fccec6467d8b45df1e58dd73df, entries=100, sequenceid=682, filesize=9.6 K 2024-12-09T01:43:21,344 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for f66467034fd235041292f78356d98c07 in 1238ms, sequenceid=682, compaction requested=true 2024-12-09T01:43:21,345 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/41692d84e39b4872a0801b4a3a224b91, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/bbe1b093667b42eaaf14b7522008f083, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/158a50d6ed954ea8973ca746c7d85dc9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/c972ed7baeb04ccfbdf8d4d3059f81f0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/3d2a14e9124145de9987b4359abe7805, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/3a856fa00abe4b729e18fff25da3bdb1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/e505587c817d40bb8e596ddbfaaaaf82, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/c939a964d0cc4a76a8ed65814c9be080, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/00c0c84d2c2541529fdd1e3d1e1a2296, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/220499b8ac544d4288a35ad0851de9ca, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/e826e9f3cb2f4006bf3048525fa5096f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/a6399a72822a48de8f3cffcf4c375125, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/f25c3dd308b7413a92df77da2da7d1e3, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/9cee9ad3224644e28f3367a6d4a2d956, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/5e3bf539c9fb4e519d3e3e6f57d6084e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/6f0282b59deb4f1fa80b78342c0a5c10, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/4270d6c139a641d78f479507d1fde502, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/f2cfae5be68d4f60920979631283fc72, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/525c47638c3a4cdf81e3ccf570d92c31, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/cfd742c58e8546da82519c19457df011, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/c1a3eaaec8504ac6a9a636bcc3b81c4b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/d9ac08f1c82e4c7e8745458135573beb, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/057f29c9558f45c8ad1191caa0152ef1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/fcd6dfd8b0e042dab06483381198c0a1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/95853d055dda447baba2803ec465982d, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/0af59de7040b4fc49995b8c866b9b50f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/2434ff6460f34fd282f7d6153810e07e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/97be3d823a7c4eaa9cd87e84bdaf786b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/ed0894dc99df40208c2879381d01a9ef, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/8dcfae2f4db547f392e727bf10cf844f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/81ea8a02df684a3da90832d159583fcb, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/bd53ce478f064ce295f2535f51f7e060, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/8ca1aabcb1964de3908b808ede2537d4, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/23be3af0ea844d4fa1d6629919b44175, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/10487e663ba24699be669e299c54709e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/45bff516de3b4cdd8313cf88aefda6c7, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/98c941355cf44e0794d656b44556f97f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/d6a288cc96f64db6b3503f8a707b84b7, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/f2eb1e7aa4c54422b3bd6f4bc9d69522] to archive 2024-12-09T01:43:21,345 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T01:43:21,346 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/41692d84e39b4872a0801b4a3a224b91 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/41692d84e39b4872a0801b4a3a224b91 2024-12-09T01:43:21,347 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/bbe1b093667b42eaaf14b7522008f083 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/bbe1b093667b42eaaf14b7522008f083 2024-12-09T01:43:21,348 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/158a50d6ed954ea8973ca746c7d85dc9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/158a50d6ed954ea8973ca746c7d85dc9 2024-12-09T01:43:21,349 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/c972ed7baeb04ccfbdf8d4d3059f81f0 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/c972ed7baeb04ccfbdf8d4d3059f81f0 2024-12-09T01:43:21,350 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/3d2a14e9124145de9987b4359abe7805 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/3d2a14e9124145de9987b4359abe7805 2024-12-09T01:43:21,351 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/3a856fa00abe4b729e18fff25da3bdb1 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/3a856fa00abe4b729e18fff25da3bdb1 2024-12-09T01:43:21,351 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/e505587c817d40bb8e596ddbfaaaaf82 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/e505587c817d40bb8e596ddbfaaaaf82 2024-12-09T01:43:21,352 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/c939a964d0cc4a76a8ed65814c9be080 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/c939a964d0cc4a76a8ed65814c9be080 2024-12-09T01:43:21,353 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/00c0c84d2c2541529fdd1e3d1e1a2296 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/00c0c84d2c2541529fdd1e3d1e1a2296 2024-12-09T01:43:21,354 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/220499b8ac544d4288a35ad0851de9ca to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/220499b8ac544d4288a35ad0851de9ca 2024-12-09T01:43:21,355 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/e826e9f3cb2f4006bf3048525fa5096f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/e826e9f3cb2f4006bf3048525fa5096f 2024-12-09T01:43:21,356 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/a6399a72822a48de8f3cffcf4c375125 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/a6399a72822a48de8f3cffcf4c375125 2024-12-09T01:43:21,357 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/f25c3dd308b7413a92df77da2da7d1e3 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/f25c3dd308b7413a92df77da2da7d1e3 2024-12-09T01:43:21,357 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/9cee9ad3224644e28f3367a6d4a2d956 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/9cee9ad3224644e28f3367a6d4a2d956 2024-12-09T01:43:21,358 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/5e3bf539c9fb4e519d3e3e6f57d6084e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/5e3bf539c9fb4e519d3e3e6f57d6084e 2024-12-09T01:43:21,359 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/6f0282b59deb4f1fa80b78342c0a5c10 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/6f0282b59deb4f1fa80b78342c0a5c10 2024-12-09T01:43:21,360 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/4270d6c139a641d78f479507d1fde502 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/4270d6c139a641d78f479507d1fde502 2024-12-09T01:43:21,360 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/f2cfae5be68d4f60920979631283fc72 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/f2cfae5be68d4f60920979631283fc72 2024-12-09T01:43:21,361 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/525c47638c3a4cdf81e3ccf570d92c31 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/525c47638c3a4cdf81e3ccf570d92c31 2024-12-09T01:43:21,364 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/cfd742c58e8546da82519c19457df011 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/cfd742c58e8546da82519c19457df011 2024-12-09T01:43:21,365 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/c1a3eaaec8504ac6a9a636bcc3b81c4b to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/c1a3eaaec8504ac6a9a636bcc3b81c4b 2024-12-09T01:43:21,365 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/d9ac08f1c82e4c7e8745458135573beb to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/d9ac08f1c82e4c7e8745458135573beb 2024-12-09T01:43:21,366 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/057f29c9558f45c8ad1191caa0152ef1 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/057f29c9558f45c8ad1191caa0152ef1 2024-12-09T01:43:21,367 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/fcd6dfd8b0e042dab06483381198c0a1 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/fcd6dfd8b0e042dab06483381198c0a1 2024-12-09T01:43:21,367 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/95853d055dda447baba2803ec465982d to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/95853d055dda447baba2803ec465982d 2024-12-09T01:43:21,368 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/0af59de7040b4fc49995b8c866b9b50f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/0af59de7040b4fc49995b8c866b9b50f 2024-12-09T01:43:21,369 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/2434ff6460f34fd282f7d6153810e07e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/2434ff6460f34fd282f7d6153810e07e 2024-12-09T01:43:21,370 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/97be3d823a7c4eaa9cd87e84bdaf786b to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/97be3d823a7c4eaa9cd87e84bdaf786b 2024-12-09T01:43:21,371 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/ed0894dc99df40208c2879381d01a9ef to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/ed0894dc99df40208c2879381d01a9ef 2024-12-09T01:43:21,372 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/8dcfae2f4db547f392e727bf10cf844f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/8dcfae2f4db547f392e727bf10cf844f 2024-12-09T01:43:21,372 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/81ea8a02df684a3da90832d159583fcb to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/81ea8a02df684a3da90832d159583fcb 2024-12-09T01:43:21,373 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/bd53ce478f064ce295f2535f51f7e060 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/bd53ce478f064ce295f2535f51f7e060 2024-12-09T01:43:21,374 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/8ca1aabcb1964de3908b808ede2537d4 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/8ca1aabcb1964de3908b808ede2537d4 2024-12-09T01:43:21,375 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/23be3af0ea844d4fa1d6629919b44175 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/23be3af0ea844d4fa1d6629919b44175 2024-12-09T01:43:21,376 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/10487e663ba24699be669e299c54709e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/10487e663ba24699be669e299c54709e 2024-12-09T01:43:21,376 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/45bff516de3b4cdd8313cf88aefda6c7 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/45bff516de3b4cdd8313cf88aefda6c7 2024-12-09T01:43:21,377 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/98c941355cf44e0794d656b44556f97f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/98c941355cf44e0794d656b44556f97f 2024-12-09T01:43:21,378 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/d6a288cc96f64db6b3503f8a707b84b7 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/d6a288cc96f64db6b3503f8a707b84b7 2024-12-09T01:43:21,379 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/f2eb1e7aa4c54422b3bd6f4bc9d69522 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/f2eb1e7aa4c54422b3bd6f4bc9d69522 2024-12-09T01:43:21,380 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/8c6f6dfa0beb46239a609eebcf3cf6d1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/4140f8b812a543efa1f6061cc9886368, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/2f8b43fd2565419bb0594e0163428a57, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/3ce7f72c851e494aa721d4c3f177579f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/a12f7098f3b0465895703717aca8ca22, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/88a4d95112834017ab9a9ba63d8d7579, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/94141d697f204096a9fe4eb189db0046, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/788a066d03a94575bc2d4a5b806e23e2, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/dfbaba299beb452d8ab7257b2d898e59, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/d916e9b2008b4e80aac8582512917d10, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/d1ea6ad0529240e2b6c9c06e3056490c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/b93de55e55fe44b88ba13d280b006bd3, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/fb31fe3ca3a84367a09d72830cbccf7a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/df5773d5fe8c48429361f144c95b9b37, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/3e3366ec799f4a71ab136a1ce364ef74, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/9941a9b380e6431aad19e2528c39281f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/51f5034fb63549debcad1ee84ed7aba4, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/5acf3ae97ec04902956022a2075d6bd7, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/6be43740ba8746bca55db2f3c2fdd7e8, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/1bafe6a87b5d4ddda8e711e2d516f41b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/8a6801a45e8c488c812f55e17446b635, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/de4ca5600e6f47dcb7935ef723946225, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/db44a3696ff54d6e9dcb97f74128519b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/882bb051af374c73aa7c2a8d03460795, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/672e8f6d29134995ab3791e71a2b28be, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/5fe7d69a54a24d74b614ab6fa9b144d7, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/0d22ad93afb4492ea0601a4aacad84ad, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/bc545062b2b54bcfa5566a4445342853, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/f665f513f1254a4eb1ff2bfceacdb6d9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/c37ae193769d43ab9fd771d9faeefeb7, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/bfebfb3563524f59a12c93ffc04ab8db, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/c3c0c48d635644d9a37fb6ecccc64f22, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/6f99fc7fb93740baa00d590e14e62f38, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/c91e7c6cfcf946388c71ec58883e904b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/97305f2811274de28d3c1cb0fb53fc8f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/5d6807a7a54c4deda091dca76492ec4a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/8cbaac29b4d745139312718add8e8dd0, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/28269c4062d346669f85cb23145b8999, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/b3b022387b904debb6be7b920afd7d35] to archive 2024-12-09T01:43:21,381 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T01:43:21,382 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/8c6f6dfa0beb46239a609eebcf3cf6d1 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/8c6f6dfa0beb46239a609eebcf3cf6d1 2024-12-09T01:43:21,383 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/4140f8b812a543efa1f6061cc9886368 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/4140f8b812a543efa1f6061cc9886368 2024-12-09T01:43:21,384 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/2f8b43fd2565419bb0594e0163428a57 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/2f8b43fd2565419bb0594e0163428a57 2024-12-09T01:43:21,384 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/3ce7f72c851e494aa721d4c3f177579f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/3ce7f72c851e494aa721d4c3f177579f 2024-12-09T01:43:21,385 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/a12f7098f3b0465895703717aca8ca22 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/a12f7098f3b0465895703717aca8ca22 2024-12-09T01:43:21,386 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/88a4d95112834017ab9a9ba63d8d7579 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/88a4d95112834017ab9a9ba63d8d7579 2024-12-09T01:43:21,387 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/94141d697f204096a9fe4eb189db0046 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/94141d697f204096a9fe4eb189db0046 2024-12-09T01:43:21,387 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/788a066d03a94575bc2d4a5b806e23e2 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/788a066d03a94575bc2d4a5b806e23e2 2024-12-09T01:43:21,388 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/dfbaba299beb452d8ab7257b2d898e59 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/dfbaba299beb452d8ab7257b2d898e59 2024-12-09T01:43:21,389 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/d916e9b2008b4e80aac8582512917d10 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/d916e9b2008b4e80aac8582512917d10 2024-12-09T01:43:21,390 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/d1ea6ad0529240e2b6c9c06e3056490c to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/d1ea6ad0529240e2b6c9c06e3056490c 2024-12-09T01:43:21,391 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/b93de55e55fe44b88ba13d280b006bd3 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/b93de55e55fe44b88ba13d280b006bd3 2024-12-09T01:43:21,392 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/fb31fe3ca3a84367a09d72830cbccf7a to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/fb31fe3ca3a84367a09d72830cbccf7a 2024-12-09T01:43:21,393 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/df5773d5fe8c48429361f144c95b9b37 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/df5773d5fe8c48429361f144c95b9b37 2024-12-09T01:43:21,393 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/3e3366ec799f4a71ab136a1ce364ef74 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/3e3366ec799f4a71ab136a1ce364ef74 2024-12-09T01:43:21,394 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/9941a9b380e6431aad19e2528c39281f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/9941a9b380e6431aad19e2528c39281f 2024-12-09T01:43:21,395 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/51f5034fb63549debcad1ee84ed7aba4 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/51f5034fb63549debcad1ee84ed7aba4 2024-12-09T01:43:21,396 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/5acf3ae97ec04902956022a2075d6bd7 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/5acf3ae97ec04902956022a2075d6bd7 2024-12-09T01:43:21,396 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/6be43740ba8746bca55db2f3c2fdd7e8 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/6be43740ba8746bca55db2f3c2fdd7e8 2024-12-09T01:43:21,397 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/1bafe6a87b5d4ddda8e711e2d516f41b to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/1bafe6a87b5d4ddda8e711e2d516f41b 2024-12-09T01:43:21,398 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/8a6801a45e8c488c812f55e17446b635 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/8a6801a45e8c488c812f55e17446b635 2024-12-09T01:43:21,399 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/de4ca5600e6f47dcb7935ef723946225 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/de4ca5600e6f47dcb7935ef723946225 2024-12-09T01:43:21,400 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/db44a3696ff54d6e9dcb97f74128519b to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/db44a3696ff54d6e9dcb97f74128519b 2024-12-09T01:43:21,400 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/882bb051af374c73aa7c2a8d03460795 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/882bb051af374c73aa7c2a8d03460795 2024-12-09T01:43:21,401 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/672e8f6d29134995ab3791e71a2b28be to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/672e8f6d29134995ab3791e71a2b28be 2024-12-09T01:43:21,402 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/5fe7d69a54a24d74b614ab6fa9b144d7 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/5fe7d69a54a24d74b614ab6fa9b144d7 2024-12-09T01:43:21,403 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/0d22ad93afb4492ea0601a4aacad84ad to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/0d22ad93afb4492ea0601a4aacad84ad 2024-12-09T01:43:21,403 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/bc545062b2b54bcfa5566a4445342853 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/bc545062b2b54bcfa5566a4445342853 2024-12-09T01:43:21,404 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/f665f513f1254a4eb1ff2bfceacdb6d9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/f665f513f1254a4eb1ff2bfceacdb6d9 2024-12-09T01:43:21,405 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/c37ae193769d43ab9fd771d9faeefeb7 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/c37ae193769d43ab9fd771d9faeefeb7 2024-12-09T01:43:21,406 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/bfebfb3563524f59a12c93ffc04ab8db to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/bfebfb3563524f59a12c93ffc04ab8db 2024-12-09T01:43:21,406 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/c3c0c48d635644d9a37fb6ecccc64f22 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/c3c0c48d635644d9a37fb6ecccc64f22 2024-12-09T01:43:21,407 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/6f99fc7fb93740baa00d590e14e62f38 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/6f99fc7fb93740baa00d590e14e62f38 2024-12-09T01:43:21,408 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/c91e7c6cfcf946388c71ec58883e904b to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/c91e7c6cfcf946388c71ec58883e904b 2024-12-09T01:43:21,408 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/97305f2811274de28d3c1cb0fb53fc8f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/97305f2811274de28d3c1cb0fb53fc8f 2024-12-09T01:43:21,409 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/5d6807a7a54c4deda091dca76492ec4a to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/5d6807a7a54c4deda091dca76492ec4a 2024-12-09T01:43:21,410 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/8cbaac29b4d745139312718add8e8dd0 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/8cbaac29b4d745139312718add8e8dd0 2024-12-09T01:43:21,411 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/28269c4062d346669f85cb23145b8999 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/28269c4062d346669f85cb23145b8999 2024-12-09T01:43:21,412 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/b3b022387b904debb6be7b920afd7d35 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/b3b022387b904debb6be7b920afd7d35 2024-12-09T01:43:21,413 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/239f8827e57c46b89c932ee63daa8e61, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/b67c2ff4f6ca447abb9bf81709d278ff, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/2176e3fc19144167a33c449aed55de40, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/9ea72d89aa4d472e8e807e21d5888f4a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/6d5ed1e31b1145b1891a4664d3e3f451, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/296f9f714a32496d8f64b0cd1fb8b2fe, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/fc406365557445d6abe027e1202f4a98, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/4a02e4623c6943dcaab4d9c10f2d0de2, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/5d55e0c812aa40b984714c5379ba8a6a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/c82ec52a3c2c4d3894f9995fe7582909, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/ec1c7f89c470422480f790cb99f97b1c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/efdbbf24f55b42569ec758bd829db0d5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/0ceaf987aa6241b986f2938c1fce6472, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/216d4e02d1764589bc00e7bae6ffcf13, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/3348f7a5a3444642b192b468dde77bcb, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/2dbd43b916cb481bb5da66730d069276, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/ab8041e56c1542309341afdaad8f673d, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/e379ba35f5e34f818bc15d8dc8df08c7, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/284e04123c0742f28823a7b6484b62e9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/b946c991062949d48631f5ea88cc448f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/21291c9fd37e4219b134f4d0527b45ec, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/bd361cef26eb4ad481907a664774510f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/b63f06cf14fd453b91be7d3cf845f1f4, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/06f1bfe5b885428688fa4aa9c44ac27a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/d1ef94dcb9034bc9b81508542a6f7e11, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/6e10f28c8bd84fa29c3b5c59a8ab849e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/6dfd4034a04a4093a4fe158bdc56c637, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/4fd713d72b394da7add201a95ad65296, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/09f585dc1064480e81c25b3b90ca450f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/f7cf284d1034435190f5799af1cb6cf5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/af27f6fe5f7643448a03aa750b940059, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/9a00dc2265fb4e988168d68f9bf04ce8, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/946945301b9e47f687e7997b7a072361, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/34005f12b8474fd2bcd611dc33f3224c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/f96778ee388b4062bedb15b44f1308f4, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/2433268f5cf044ccbf044d75770026b7] to archive 2024-12-09T01:43:21,414 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T01:43:21,415 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/239f8827e57c46b89c932ee63daa8e61 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/239f8827e57c46b89c932ee63daa8e61 2024-12-09T01:43:21,416 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/b67c2ff4f6ca447abb9bf81709d278ff to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/b67c2ff4f6ca447abb9bf81709d278ff 2024-12-09T01:43:21,417 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/2176e3fc19144167a33c449aed55de40 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/2176e3fc19144167a33c449aed55de40 2024-12-09T01:43:21,418 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/9ea72d89aa4d472e8e807e21d5888f4a to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/9ea72d89aa4d472e8e807e21d5888f4a 2024-12-09T01:43:21,419 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/6d5ed1e31b1145b1891a4664d3e3f451 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/6d5ed1e31b1145b1891a4664d3e3f451 2024-12-09T01:43:21,420 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/296f9f714a32496d8f64b0cd1fb8b2fe to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/296f9f714a32496d8f64b0cd1fb8b2fe 2024-12-09T01:43:21,421 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/fc406365557445d6abe027e1202f4a98 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/fc406365557445d6abe027e1202f4a98 2024-12-09T01:43:21,422 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/4a02e4623c6943dcaab4d9c10f2d0de2 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/4a02e4623c6943dcaab4d9c10f2d0de2 2024-12-09T01:43:21,423 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/5d55e0c812aa40b984714c5379ba8a6a to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/5d55e0c812aa40b984714c5379ba8a6a 2024-12-09T01:43:21,423 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/c82ec52a3c2c4d3894f9995fe7582909 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/c82ec52a3c2c4d3894f9995fe7582909 2024-12-09T01:43:21,424 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/ec1c7f89c470422480f790cb99f97b1c to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/ec1c7f89c470422480f790cb99f97b1c 2024-12-09T01:43:21,425 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/efdbbf24f55b42569ec758bd829db0d5 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/efdbbf24f55b42569ec758bd829db0d5 2024-12-09T01:43:21,426 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/0ceaf987aa6241b986f2938c1fce6472 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/0ceaf987aa6241b986f2938c1fce6472 2024-12-09T01:43:21,427 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/216d4e02d1764589bc00e7bae6ffcf13 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/216d4e02d1764589bc00e7bae6ffcf13 2024-12-09T01:43:21,428 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/3348f7a5a3444642b192b468dde77bcb to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/3348f7a5a3444642b192b468dde77bcb 2024-12-09T01:43:21,429 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/2dbd43b916cb481bb5da66730d069276 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/2dbd43b916cb481bb5da66730d069276 2024-12-09T01:43:21,430 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/ab8041e56c1542309341afdaad8f673d to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/ab8041e56c1542309341afdaad8f673d 2024-12-09T01:43:21,430 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/e379ba35f5e34f818bc15d8dc8df08c7 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/e379ba35f5e34f818bc15d8dc8df08c7 2024-12-09T01:43:21,431 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/284e04123c0742f28823a7b6484b62e9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/284e04123c0742f28823a7b6484b62e9 2024-12-09T01:43:21,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/b946c991062949d48631f5ea88cc448f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/b946c991062949d48631f5ea88cc448f 2024-12-09T01:43:21,433 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/21291c9fd37e4219b134f4d0527b45ec to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/21291c9fd37e4219b134f4d0527b45ec 2024-12-09T01:43:21,434 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/bd361cef26eb4ad481907a664774510f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/bd361cef26eb4ad481907a664774510f 2024-12-09T01:43:21,435 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/b63f06cf14fd453b91be7d3cf845f1f4 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/b63f06cf14fd453b91be7d3cf845f1f4 2024-12-09T01:43:21,436 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/06f1bfe5b885428688fa4aa9c44ac27a to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/06f1bfe5b885428688fa4aa9c44ac27a 2024-12-09T01:43:21,436 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/d1ef94dcb9034bc9b81508542a6f7e11 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/d1ef94dcb9034bc9b81508542a6f7e11 2024-12-09T01:43:21,437 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/6e10f28c8bd84fa29c3b5c59a8ab849e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/6e10f28c8bd84fa29c3b5c59a8ab849e 2024-12-09T01:43:21,438 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/6dfd4034a04a4093a4fe158bdc56c637 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/6dfd4034a04a4093a4fe158bdc56c637 2024-12-09T01:43:21,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/4fd713d72b394da7add201a95ad65296 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/4fd713d72b394da7add201a95ad65296 2024-12-09T01:43:21,440 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/09f585dc1064480e81c25b3b90ca450f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/09f585dc1064480e81c25b3b90ca450f 2024-12-09T01:43:21,440 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/f7cf284d1034435190f5799af1cb6cf5 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/f7cf284d1034435190f5799af1cb6cf5 2024-12-09T01:43:21,441 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/af27f6fe5f7643448a03aa750b940059 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/af27f6fe5f7643448a03aa750b940059 2024-12-09T01:43:21,442 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/9a00dc2265fb4e988168d68f9bf04ce8 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/9a00dc2265fb4e988168d68f9bf04ce8 2024-12-09T01:43:21,443 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/946945301b9e47f687e7997b7a072361 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/946945301b9e47f687e7997b7a072361 2024-12-09T01:43:21,443 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/34005f12b8474fd2bcd611dc33f3224c to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/34005f12b8474fd2bcd611dc33f3224c 2024-12-09T01:43:21,444 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/f96778ee388b4062bedb15b44f1308f4 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/f96778ee388b4062bedb15b44f1308f4 2024-12-09T01:43:21,445 DEBUG [StoreCloser-TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/2433268f5cf044ccbf044d75770026b7 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/2433268f5cf044ccbf044d75770026b7 2024-12-09T01:43:21,449 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/recovered.edits/685.seqid, newMaxSeqId=685, maxSeqId=1 2024-12-09T01:43:21,450 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. 2024-12-09T01:43:21,450 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for f66467034fd235041292f78356d98c07: Waiting for close lock at 1733708599819Running coprocessor pre-close hooks at 1733708599819Disabling compacts and flushes for region at 1733708599819Disabling writes for close at 1733708600106 (+287 ms)Obtaining lock to block concurrent updates at 1733708600106Preparing flush snapshotting stores in f66467034fd235041292f78356d98c07 at 1733708600106Finished memstore snapshotting TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733708600107 (+1 ms)Flushing stores of TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07. at 1733708600107Flushing f66467034fd235041292f78356d98c07/A: creating writer at 1733708600107Flushing f66467034fd235041292f78356d98c07/A: appending metadata at 1733708600109 (+2 ms)Flushing f66467034fd235041292f78356d98c07/A: closing flushed file at 1733708600109Flushing f66467034fd235041292f78356d98c07/B: creating writer at 1733708600516 (+407 ms)Flushing f66467034fd235041292f78356d98c07/B: appending metadata at 1733708600518 (+2 ms)Flushing f66467034fd235041292f78356d98c07/B: closing flushed file at 1733708600518Flushing f66467034fd235041292f78356d98c07/C: creating writer at 1733708600926 (+408 ms)Flushing f66467034fd235041292f78356d98c07/C: appending metadata at 1733708600927 (+1 ms)Flushing f66467034fd235041292f78356d98c07/C: closing flushed file at 1733708600928 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@252dcb66: reopening flushed file at 1733708601334 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d51eeb0: reopening flushed file at 1733708601337 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@65ac617f: reopening flushed file at 1733708601340 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for f66467034fd235041292f78356d98c07 in 1238ms, sequenceid=682, compaction requested=true at 1733708601344 (+4 ms)Writing region close event to WAL at 1733708601446 (+102 ms)Running coprocessor post-close hooks at 1733708601450 (+4 ms)Closed at 1733708601450 2024-12-09T01:43:21,452 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed f66467034fd235041292f78356d98c07 2024-12-09T01:43:21,452 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=f66467034fd235041292f78356d98c07, regionState=CLOSED 2024-12-09T01:43:21,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 because future has completed 2024-12-09T01:43:21,456 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-09T01:43:21,456 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure f66467034fd235041292f78356d98c07, server=ef6f18c58dc9,38627,1733708453943 in 1.7880 sec 2024-12-09T01:43:21,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-12-09T01:43:21,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f66467034fd235041292f78356d98c07, UNASSIGN in 1.7930 sec 2024-12-09T01:43:21,459 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-09T01:43:21,459 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.7960 sec 2024-12-09T01:43:21,460 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708601460"}]},"ts":"1733708601460"} 2024-12-09T01:43:21,461 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T01:43:21,461 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T01:43:21,463 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8040 sec 2024-12-09T01:43:21,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-09T01:43:21,803 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:21,804 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T01:43:21,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:43:21,806 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:43:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-09T01:43:21,806 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:43:21,809 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07 2024-12-09T01:43:21,811 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A, FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B, FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C, FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/recovered.edits] 2024-12-09T01:43:21,813 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/0a6dbb1ae0254e71a045dac0937b0d26 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/0a6dbb1ae0254e71a045dac0937b0d26 2024-12-09T01:43:21,814 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/0cf4512711a64d998783e96070fa39c8 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/A/0cf4512711a64d998783e96070fa39c8 2024-12-09T01:43:21,815 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/1a22bb709af340679676da9a1bc81e91 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/1a22bb709af340679676da9a1bc81e91 2024-12-09T01:43:21,816 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/cedb0dd0b7d44e259c17623eea14271f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/B/cedb0dd0b7d44e259c17623eea14271f 2024-12-09T01:43:21,818 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/88b58205d8eb4f72a0da628d10c0968f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/88b58205d8eb4f72a0da628d10c0968f 2024-12-09T01:43:21,819 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/9070807c3d2c47b58e5ea8f74d864be0 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/9070807c3d2c47b58e5ea8f74d864be0 2024-12-09T01:43:21,820 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/9304149a1b60435f8628c71047255a41 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/9304149a1b60435f8628c71047255a41 2024-12-09T01:43:21,820 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/9f18a7fccec6467d8b45df1e58dd73df to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/C/9f18a7fccec6467d8b45df1e58dd73df 2024-12-09T01:43:21,822 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/recovered.edits/685.seqid to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07/recovered.edits/685.seqid 2024-12-09T01:43:21,823 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/f66467034fd235041292f78356d98c07 2024-12-09T01:43:21,823 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T01:43:21,825 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:43:21,826 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T01:43:21,828 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T01:43:21,829 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:43:21,829 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T01:43:21,830 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733708601829"}]},"ts":"9223372036854775807"} 2024-12-09T01:43:21,831 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T01:43:21,831 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f66467034fd235041292f78356d98c07, NAME => 'TestAcidGuarantees,,1733708575333.f66467034fd235041292f78356d98c07.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T01:43:21,831 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T01:43:21,832 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733708601831"}]},"ts":"9223372036854775807"} 2024-12-09T01:43:21,833 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T01:43:21,834 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:43:21,835 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 30 msec 2024-12-09T01:43:21,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-09T01:43:21,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T01:43:21,914 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:21,923 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=253 (was 251) - Thread LEAK? -, OpenFileDescriptor=461 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=673 (was 594) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=11171 (was 11224) 2024-12-09T01:43:21,931 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=253, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=673, ProcessCount=11, AvailableMemoryMB=11170 2024-12-09T01:43:21,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T01:43:21,933 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:43:21,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-12-09T01:43:21,934 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T01:43:21,935 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:21,935 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-12-09T01:43:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-09T01:43:21,935 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T01:43:21,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742429_1605 (size=1061) 2024-12-09T01:43:21,959 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T01:43:22,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-09T01:43:22,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-09T01:43:22,341 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb 2024-12-09T01:43:22,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742430_1606 (size=53) 2024-12-09T01:43:22,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-09T01:43:22,746 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:43:22,746 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 45d234ddf430a4159b8bc5324edd88bc, disabling compactions & flushes 2024-12-09T01:43:22,746 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:22,746 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:22,746 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. after waiting 0 ms 2024-12-09T01:43:22,746 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:22,746 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:22,747 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 45d234ddf430a4159b8bc5324edd88bc: Waiting for close lock at 1733708602746Disabling compacts and flushes for region at 1733708602746Disabling writes for close at 1733708602746Writing region close event to WAL at 1733708602746Closed at 1733708602746 2024-12-09T01:43:22,748 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T01:43:22,748 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1733708602748"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733708602748"}]},"ts":"1733708602748"} 2024-12-09T01:43:22,749 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-09T01:43:22,750 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T01:43:22,750 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708602750"}]},"ts":"1733708602750"} 2024-12-09T01:43:22,751 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-12-09T01:43:22,751 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45d234ddf430a4159b8bc5324edd88bc, ASSIGN}] 2024-12-09T01:43:22,752 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45d234ddf430a4159b8bc5324edd88bc, ASSIGN 2024-12-09T01:43:22,752 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45d234ddf430a4159b8bc5324edd88bc, ASSIGN; state=OFFLINE, location=ef6f18c58dc9,38627,1733708453943; forceNewPlan=false, retain=false 2024-12-09T01:43:22,903 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=45d234ddf430a4159b8bc5324edd88bc, regionState=OPENING, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:22,905 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45d234ddf430a4159b8bc5324edd88bc, ASSIGN because future has completed 2024-12-09T01:43:22,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943}] 2024-12-09T01:43:23,059 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:23,059 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:43:23,060 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:23,060 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:43:23,060 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:23,060 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:23,061 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:23,062 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:43:23,062 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45d234ddf430a4159b8bc5324edd88bc columnFamilyName A 2024-12-09T01:43:23,062 DEBUG [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:23,063 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] regionserver.HStore(327): Store=45d234ddf430a4159b8bc5324edd88bc/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:43:23,063 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:23,063 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:43:23,064 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45d234ddf430a4159b8bc5324edd88bc columnFamilyName B 2024-12-09T01:43:23,064 DEBUG [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:23,064 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] regionserver.HStore(327): Store=45d234ddf430a4159b8bc5324edd88bc/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:43:23,064 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:23,065 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:43:23,065 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45d234ddf430a4159b8bc5324edd88bc columnFamilyName C 2024-12-09T01:43:23,065 DEBUG [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:23,065 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] regionserver.HStore(327): Store=45d234ddf430a4159b8bc5324edd88bc/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:43:23,065 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:23,065 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:23,066 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:23,066 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:23,067 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:23,067 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:23,067 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T01:43:23,068 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:23,070 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T01:43:23,070 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened 45d234ddf430a4159b8bc5324edd88bc; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66383153, jitterRate=-0.010813936591148376}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T01:43:23,070 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:23,071 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for 45d234ddf430a4159b8bc5324edd88bc: Running coprocessor pre-open hook at 1733708603060Writing region info on filesystem at 1733708603060Initializing all the Stores at 1733708603060Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708603060Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708603061 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708603061Cleaning up temporary data from old regions at 1733708603067 (+6 ms)Running coprocessor post-open hooks at 1733708603070 (+3 ms)Region opened successfully at 1733708603071 (+1 ms) 2024-12-09T01:43:23,072 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., pid=146, masterSystemTime=1733708603056 2024-12-09T01:43:23,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-09T01:43:23,073 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:23,073 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:23,074 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=45d234ddf430a4159b8bc5324edd88bc, regionState=OPEN, openSeqNum=2, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:23,075 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 because future has completed 2024-12-09T01:43:23,077 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-12-09T01:43:23,077 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 in 170 msec 2024-12-09T01:43:23,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-12-09T01:43:23,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45d234ddf430a4159b8bc5324edd88bc, ASSIGN in 326 msec 2024-12-09T01:43:23,079 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T01:43:23,079 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708603079"}]},"ts":"1733708603079"} 2024-12-09T01:43:23,080 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-12-09T01:43:23,081 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T01:43:23,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1480 sec 2024-12-09T01:43:24,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-12-09T01:43:24,084 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:24,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@237103d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:24,085 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:43:24,085 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:43:24,086 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:43:24,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:43:24,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:43:24,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3117b36c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:24,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:43:24,086 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:43:24,087 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:24,087 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46908, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:43:24,088 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18b5f7ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:24,088 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:43:24,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:43:24,089 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:43:24,090 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:43:24,091 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-09T01:43:24,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is ef6f18c58dc9,37995,1733708453073 2024-12-09T01:43:24,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5fcf7cc4 2024-12-09T01:43:24,091 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T01:43:24,092 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T01:43:24,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-12-09T01:43:24,093 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T01:43:24,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-12-09T01:43:24,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742431_1607 (size=1097) 2024-12-09T01:43:24,504 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-12-09T01:43:24,504 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-12-09T01:43:24,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T01:43:24,509 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45d234ddf430a4159b8bc5324edd88bc, REOPEN/MOVE}] 2024-12-09T01:43:24,509 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45d234ddf430a4159b8bc5324edd88bc, REOPEN/MOVE 2024-12-09T01:43:24,510 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=45d234ddf430a4159b8bc5324edd88bc, regionState=CLOSING, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:24,511 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45d234ddf430a4159b8bc5324edd88bc, REOPEN/MOVE because future has completed 2024-12-09T01:43:24,511 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T01:43:24,511 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943}] 2024-12-09T01:43:24,663 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:24,663 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T01:43:24,663 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 45d234ddf430a4159b8bc5324edd88bc, disabling compactions & flushes 2024-12-09T01:43:24,663 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:24,663 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:24,663 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. after waiting 0 ms 2024-12-09T01:43:24,663 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:24,667 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-12-09T01:43:24,667 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:24,667 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 45d234ddf430a4159b8bc5324edd88bc: Waiting for close lock at 1733708604663Running coprocessor pre-close hooks at 1733708604663Disabling compacts and flushes for region at 1733708604663Disabling writes for close at 1733708604663Writing region close event to WAL at 1733708604664 (+1 ms)Running coprocessor post-close hooks at 1733708604667 (+3 ms)Closed at 1733708604667 2024-12-09T01:43:24,667 WARN [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: 45d234ddf430a4159b8bc5324edd88bc to self. 2024-12-09T01:43:24,669 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:24,669 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=45d234ddf430a4159b8bc5324edd88bc, regionState=CLOSED 2024-12-09T01:43:24,671 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 because future has completed 2024-12-09T01:43:24,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-12-09T01:43:24,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 in 160 msec 2024-12-09T01:43:24,673 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45d234ddf430a4159b8bc5324edd88bc, REOPEN/MOVE; state=CLOSED, location=ef6f18c58dc9,38627,1733708453943; forceNewPlan=false, retain=true 2024-12-09T01:43:24,824 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=45d234ddf430a4159b8bc5324edd88bc, regionState=OPENING, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:24,825 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45d234ddf430a4159b8bc5324edd88bc, REOPEN/MOVE because future has completed 2024-12-09T01:43:24,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943}] 2024-12-09T01:43:24,980 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:24,980 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} 2024-12-09T01:43:24,980 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:24,980 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T01:43:24,980 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:24,980 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:24,981 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:24,982 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:43:24,982 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45d234ddf430a4159b8bc5324edd88bc columnFamilyName A 2024-12-09T01:43:24,983 DEBUG [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:24,983 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] regionserver.HStore(327): Store=45d234ddf430a4159b8bc5324edd88bc/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:43:24,983 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:24,984 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:43:24,984 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45d234ddf430a4159b8bc5324edd88bc columnFamilyName B 2024-12-09T01:43:24,984 DEBUG [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:24,984 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] regionserver.HStore(327): Store=45d234ddf430a4159b8bc5324edd88bc/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:43:24,984 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:24,985 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-12-09T01:43:24,985 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45d234ddf430a4159b8bc5324edd88bc columnFamilyName C 2024-12-09T01:43:24,985 DEBUG [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:24,985 INFO [StoreOpener-45d234ddf430a4159b8bc5324edd88bc-1 {}] regionserver.HStore(327): Store=45d234ddf430a4159b8bc5324edd88bc/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T01:43:24,985 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:24,985 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:24,986 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:24,986 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:24,987 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:24,987 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:24,987 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-09T01:43:24,988 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:24,989 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened 45d234ddf430a4159b8bc5324edd88bc; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64682385, jitterRate=-0.03615735471248627}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-09T01:43:24,989 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:24,989 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for 45d234ddf430a4159b8bc5324edd88bc: Running coprocessor pre-open hook at 1733708604980Writing region info on filesystem at 1733708604980Initializing all the Stores at 1733708604981 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708604981Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708604981Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733708604981Cleaning up temporary data from old regions at 1733708604987 (+6 ms)Running coprocessor post-open hooks at 1733708604989 (+2 ms)Region opened successfully at 1733708604989 2024-12-09T01:43:24,990 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., pid=151, masterSystemTime=1733708604977 2024-12-09T01:43:24,991 DEBUG [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:24,991 INFO [RS_OPEN_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:24,992 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=45d234ddf430a4159b8bc5324edd88bc, regionState=OPEN, openSeqNum=5, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:24,993 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 because future has completed 2024-12-09T01:43:24,995 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-12-09T01:43:24,995 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 in 169 msec 2024-12-09T01:43:24,996 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-12-09T01:43:24,996 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45d234ddf430a4159b8bc5324edd88bc, REOPEN/MOVE in 487 msec 2024-12-09T01:43:24,999 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-12-09T01:43:24,999 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 490 msec 2024-12-09T01:43:25,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 905 msec 2024-12-09T01:43:25,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-12-09T01:43:25,001 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:25,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4826bbfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:43:25,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:43:25,003 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:43:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:43:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:43:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@784f065f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:43:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:43:25,004 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:25,004 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:43:25,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e7205a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,006 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac41996, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,006 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:43:25,006 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:43:25,006 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:43:25,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:43:25,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:43:25,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d34ae03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:43:25,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:43:25,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:25,008 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46942, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:43:25,008 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76d6def7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22aab9aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,009 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:43:25,009 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:43:25,010 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:43:25,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:43:25,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:43:25,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c64741c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:43:25,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:43:25,010 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:25,011 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:43:25,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5857f2f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a34ec22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,012 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:43:25,012 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:43:25,013 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:43:25,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:43:25,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:43:25,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e71c43a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:43:25,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:43:25,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:25,014 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:43:25,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@229db3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@380ce272, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,015 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:43:25,015 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:43:25,015 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:43:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:43:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:43:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3aefadc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:43:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:43:25,016 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:25,017 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:43:25,017 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15346ce8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4885acdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,018 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:43:25,018 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:43:25,018 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:43:25,018 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:43:25,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:43:25,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e5e8a79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:43:25,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:43:25,019 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:25,019 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:43:25,020 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46989ecd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,020 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cc99027, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,020 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:43:25,021 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:43:25,021 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:43:25,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:43:25,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:43:25,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f5bb66f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,021 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:43:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:43:25,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:25,022 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:43:25,023 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14866f03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24edf963, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,023 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:43:25,023 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:43:25,024 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:43:25,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:43:25,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:43:25,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2444274e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:43:25,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:43:25,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:25,025 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:43:25,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@126c11f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,026 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2966c360, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,026 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:43:25,026 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:43:25,027 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:43:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:43:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:43:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5400c37e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:43:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:43:25,027 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:25,028 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:43:25,028 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af74f69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,029 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6379f6de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,029 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ef6f18c58dc9,37995,-1 for getting cluster id 2024-12-09T01:43:25,029 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-09T01:43:25,030 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '839c1de9-5c28-4534-95ef-dc24ebf74dc2' 2024-12-09T01:43:25,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-09T01:43:25,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "839c1de9-5c28-4534-95ef-dc24ebf74dc2" 2024-12-09T01:43:25,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17df1f85, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ef6f18c58dc9,37995,-1] 2024-12-09T01:43:25,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-09T01:43:25,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:25,030 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-09T01:43:25,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41507559, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T01:43:25,031 DEBUG [Thread-2687 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:43:25,031 DEBUG [Thread-2676 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:43:25,031 DEBUG [Thread-2691 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:43:25,032 DEBUG [Thread-2682 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:43:25,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:43:25,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:43:25,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:43:25,032 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:43:25,032 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:43:25,032 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:43:25,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:43:25,033 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:43:25,033 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:43:25,033 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:43:25,033 DEBUG [Thread-2674 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:43:25,034 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39966, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:43:25,034 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:43:25,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:43:25,034 DEBUG [Thread-2685 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:43:25,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:43:25,035 DEBUG [Thread-2680 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:43:25,035 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:43:25,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:43:25,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:43:25,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:43:25,036 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:43:25,036 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39992, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:43:25,036 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:43:25,036 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40008, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:43:25,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:43:25,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:43:25,037 DEBUG [Thread-2689 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:43:25,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:43:25,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:43:25,038 DEBUG [Thread-2678 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:43:25,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:43:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:43:25,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-12-09T01:43:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:43:25,039 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:43:25,039 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:43:25,039 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:43:25,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T01:43:25,040 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:43:25,040 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:43:25,040 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40044, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:43:25,041 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:43:25,041 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:43:25,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:43:25,044 DEBUG [Thread-2693 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-09T01:43:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:43:25,045 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:43:25,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ef6f18c58dc9,38627,1733708453943, seqNum=-1] 2024-12-09T01:43:25,045 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T01:43:25,046 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T01:43:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:25,048 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5] 2024-12-09T01:43:25,048 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45d234ddf430a4159b8bc5324edd88bc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:43:25,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=A 2024-12-09T01:43:25,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:25,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=B 2024-12-09T01:43:25,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:25,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=C 2024-12-09T01:43:25,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:25,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209baa30410c76f4ddcab2038ad71aea018_45d234ddf430a4159b8bc5324edd88bc is 50, key is test_row_0/A:col10/1733708605047/Put/seqid=0 2024-12-09T01:43:25,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:25,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708665077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:25,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708665077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,085 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:25,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708665083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:25,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708665085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:25,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:25,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,091 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708665085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:25,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,091 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:25,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:25,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742432_1608 (size=17034) 2024-12-09T01:43:25,098 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:25,101 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209baa30410c76f4ddcab2038ad71aea018_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209baa30410c76f4ddcab2038ad71aea018_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:25,103 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/ca9f66c95fec4d87a8129559a638fbc7, store: [table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:25,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/ca9f66c95fec4d87a8129559a638fbc7 is 175, key is test_row_0/A:col10/1733708605047/Put/seqid=0 2024-12-09T01:43:25,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742433_1609 (size=48139) 2024-12-09T01:43:25,128 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/ca9f66c95fec4d87a8129559a638fbc7 2024-12-09T01:43:25,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T01:43:25,155 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/5c642edf9e9f44baa1cadf1bbd33d425 is 50, key is test_row_0/B:col10/1733708605047/Put/seqid=0 2024-12-09T01:43:25,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742434_1610 (size=12001) 2024-12-09T01:43:25,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T01:43:25,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:25,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:25,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:25,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:25,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:25,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:25,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708665194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:25,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:25,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708665194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:25,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708665194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:25,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708665195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708665195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,198 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,199 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:25,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T01:43:25,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:25,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:25,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:25,347 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:25,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:25,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:25,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T01:43:25,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:25,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708665403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:25,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:25,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708665403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:25,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708665403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:25,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708665403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708665404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:25,407 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:25,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,408 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:25,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:25,501 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T01:43:25,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:25,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:25,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:25,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:25,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:25,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:25,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/5c642edf9e9f44baa1cadf1bbd33d425 2024-12-09T01:43:25,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/a8c30e8d21564c51b22f8d56af71f732 is 50, key is test_row_0/C:col10/1733708605047/Put/seqid=0 2024-12-09T01:43:25,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742435_1611 (size=12001) 2024-12-09T01:43:25,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T01:43:25,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:25,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:25,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:25,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:25,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:25,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T01:43:25,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708665713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:25,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:25,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708665714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:25,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:25,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:25,717 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708665714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:25,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:25,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:25,718 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708665714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708665714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:25,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:25,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:25,718 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:25,807 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T01:43:25,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:25,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:25,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:25,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:25,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:25,960 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T01:43:25,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:25,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:25,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:25,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:25,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:25,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:25,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/a8c30e8d21564c51b22f8d56af71f732 2024-12-09T01:43:26,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/ca9f66c95fec4d87a8129559a638fbc7 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/ca9f66c95fec4d87a8129559a638fbc7 2024-12-09T01:43:26,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/ca9f66c95fec4d87a8129559a638fbc7, entries=250, sequenceid=17, filesize=47.0 K 2024-12-09T01:43:26,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/5c642edf9e9f44baa1cadf1bbd33d425 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/5c642edf9e9f44baa1cadf1bbd33d425 2024-12-09T01:43:26,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/5c642edf9e9f44baa1cadf1bbd33d425, entries=150, sequenceid=17, filesize=11.7 K 2024-12-09T01:43:26,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/a8c30e8d21564c51b22f8d56af71f732 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/a8c30e8d21564c51b22f8d56af71f732 2024-12-09T01:43:26,013 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/a8c30e8d21564c51b22f8d56af71f732, entries=150, sequenceid=17, filesize=11.7 K 2024-12-09T01:43:26,014 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 45d234ddf430a4159b8bc5324edd88bc in 966ms, sequenceid=17, compaction requested=false 2024-12-09T01:43:26,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:26,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-12-09T01:43:26,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:26,114 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 45d234ddf430a4159b8bc5324edd88bc 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T01:43:26,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=A 2024-12-09T01:43:26,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:26,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=B 2024-12-09T01:43:26,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:26,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=C 2024-12-09T01:43:26,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:26,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092e1f4839b73746868a7d97c3fbca4520_45d234ddf430a4159b8bc5324edd88bc is 50, key is test_row_0/A:col10/1733708605080/Put/seqid=0 2024-12-09T01:43:26,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742436_1612 (size=12154) 2024-12-09T01:43:26,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,159 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092e1f4839b73746868a7d97c3fbca4520_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092e1f4839b73746868a7d97c3fbca4520_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:26,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/93bfc537b6ed4c019708848a726eb884, store: [table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:26,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/93bfc537b6ed4c019708848a726eb884 is 175, key is test_row_0/A:col10/1733708605080/Put/seqid=0 2024-12-09T01:43:26,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742437_1613 (size=30955) 2024-12-09T01:43:26,166 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/93bfc537b6ed4c019708848a726eb884 2024-12-09T01:43:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T01:43:26,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/327059eb6d6b43c5978284b4274632e8 is 50, key is test_row_0/B:col10/1733708605080/Put/seqid=0 2024-12-09T01:43:26,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742438_1614 (size=12001) 2024-12-09T01:43:26,178 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/327059eb6d6b43c5978284b4274632e8 2024-12-09T01:43:26,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/94421a0b2a9044488603ce38c98e5fe1 is 50, key is test_row_0/C:col10/1733708605080/Put/seqid=0 2024-12-09T01:43:26,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742439_1615 (size=12001) 2024-12-09T01:43:26,195 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/94421a0b2a9044488603ce38c98e5fe1 2024-12-09T01:43:26,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/93bfc537b6ed4c019708848a726eb884 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/93bfc537b6ed4c019708848a726eb884 2024-12-09T01:43:26,206 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/93bfc537b6ed4c019708848a726eb884, entries=150, sequenceid=36, filesize=30.2 K 2024-12-09T01:43:26,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/327059eb6d6b43c5978284b4274632e8 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/327059eb6d6b43c5978284b4274632e8 2024-12-09T01:43:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,210 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/327059eb6d6b43c5978284b4274632e8, entries=150, sequenceid=36, filesize=11.7 K 2024-12-09T01:43:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/94421a0b2a9044488603ce38c98e5fe1 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/94421a0b2a9044488603ce38c98e5fe1 2024-12-09T01:43:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,215 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/94421a0b2a9044488603ce38c98e5fe1, entries=150, sequenceid=36, filesize=11.7 K 2024-12-09T01:43:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,215 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 45d234ddf430a4159b8bc5324edd88bc in 101ms, sequenceid=36, compaction requested=false 2024-12-09T01:43:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:26,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:26,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-12-09T01:43:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-12-09T01:43:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-12-09T01:43:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1760 sec 2024-12-09T01:43:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.1850 sec 2024-12-09T01:43:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:26,255 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45d234ddf430a4159b8bc5324edd88bc 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-12-09T01:43:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=A 2024-12-09T01:43:26,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:26,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=B 2024-12-09T01:43:26,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=C 2024-12-09T01:43:26,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209edc9d0e911ba45a1b876d47d71579cf0_45d234ddf430a4159b8bc5324edd88bc is 50, key is test_row_0/A:col10/1733708606254/Put/seqid=0 2024-12-09T01:43:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742440_1616 (size=21918) 2024-12-09T01:43:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1733708606275, max timestamp=1733708606286Num uniques 859; 2024-12-09T01:43:26,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T01:43:26,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1733708606275, max timestamp=1733708606286Num uniques 859; 2024-12-09T01:43:26,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T01:43:26,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1733708606275, max timestamp=1733708606286Num uniques 859; 2024-12-09T01:43:26,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-12-09T01:43:26,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708666310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:26,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,320 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:26,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708666316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:26,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:26,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708666317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:26,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708666319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:26,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,326 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:26,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,326 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:26,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708666321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:26,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708666423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:26,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708666423, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:26,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,427 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:26,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:26,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708666434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:26,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:26,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708666434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:26,439 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:26,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708666434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:26,439 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,439 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,439 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:26,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:26,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:26,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708666633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:26,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708666634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:26,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:26,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:26,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708666644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:26,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708666644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:26,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708666644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:26,667 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:26,671 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209edc9d0e911ba45a1b876d47d71579cf0_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209edc9d0e911ba45a1b876d47d71579cf0_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:26,672 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/f9024eebdc214a9fb7a2c2eb4bffcc68, store: [table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:26,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/f9024eebdc214a9fb7a2c2eb4bffcc68 is 175, key is test_row_0/A:col10/1733708606254/Put/seqid=0 2024-12-09T01:43:26,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742441_1617 (size=65323) 2024-12-09T01:43:26,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708666944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:26,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708666944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:26,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708666954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:26,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:26,956 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708666954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:26,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708666954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:26,965 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T01:43:27,077 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=46, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/f9024eebdc214a9fb7a2c2eb4bffcc68 2024-12-09T01:43:27,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/3d3bc86f39254fa1a70dac7f29714278 is 50, key is test_row_0/B:col10/1733708606254/Put/seqid=0 2024-12-09T01:43:27,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742442_1618 (size=12001) 2024-12-09T01:43:27,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/3d3bc86f39254fa1a70dac7f29714278 2024-12-09T01:43:27,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/4a09a250d7a84389974b1b6d74090a1d is 50, key is test_row_0/C:col10/1733708606254/Put/seqid=0 2024-12-09T01:43:27,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742443_1619 (size=12001) 2024-12-09T01:43:27,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=46 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/4a09a250d7a84389974b1b6d74090a1d 2024-12-09T01:43:27,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/f9024eebdc214a9fb7a2c2eb4bffcc68 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/f9024eebdc214a9fb7a2c2eb4bffcc68 2024-12-09T01:43:27,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/f9024eebdc214a9fb7a2c2eb4bffcc68, entries=350, sequenceid=46, filesize=63.8 K 2024-12-09T01:43:27,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/3d3bc86f39254fa1a70dac7f29714278 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/3d3bc86f39254fa1a70dac7f29714278 2024-12-09T01:43:27,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/3d3bc86f39254fa1a70dac7f29714278, entries=150, sequenceid=46, filesize=11.7 K 2024-12-09T01:43:27,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/4a09a250d7a84389974b1b6d74090a1d as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/4a09a250d7a84389974b1b6d74090a1d 2024-12-09T01:43:27,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/4a09a250d7a84389974b1b6d74090a1d, entries=150, sequenceid=46, filesize=11.7 K 2024-12-09T01:43:27,128 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 45d234ddf430a4159b8bc5324edd88bc in 873ms, sequenceid=46, compaction requested=true 2024-12-09T01:43:27,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:27,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:43:27,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:27,128 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:27,128 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:43:27,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:27,129 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:27,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:43:27,129 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:27,129 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 144417 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:27,129 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:27,129 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/A is initiating minor compaction (all files) 2024-12-09T01:43:27,129 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/B is initiating minor compaction (all files) 2024-12-09T01:43:27,129 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/B in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:27,130 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/5c642edf9e9f44baa1cadf1bbd33d425, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/327059eb6d6b43c5978284b4274632e8, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/3d3bc86f39254fa1a70dac7f29714278] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=35.2 K 2024-12-09T01:43:27,130 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/A in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:27,130 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/ca9f66c95fec4d87a8129559a638fbc7, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/93bfc537b6ed4c019708848a726eb884, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/f9024eebdc214a9fb7a2c2eb4bffcc68] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=141.0 K 2024-12-09T01:43:27,130 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:27,130 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. files: [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/ca9f66c95fec4d87a8129559a638fbc7, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/93bfc537b6ed4c019708848a726eb884, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/f9024eebdc214a9fb7a2c2eb4bffcc68] 2024-12-09T01:43:27,131 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c642edf9e9f44baa1cadf1bbd33d425, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733708605046 2024-12-09T01:43:27,131 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca9f66c95fec4d87a8129559a638fbc7, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733708605044 2024-12-09T01:43:27,131 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 327059eb6d6b43c5978284b4274632e8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733708605073 2024-12-09T01:43:27,131 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93bfc537b6ed4c019708848a726eb884, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733708605073 2024-12-09T01:43:27,131 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d3bc86f39254fa1a70dac7f29714278, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733708606243 2024-12-09T01:43:27,132 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9024eebdc214a9fb7a2c2eb4bffcc68, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733708606241 2024-12-09T01:43:27,138 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:27,139 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#B#compaction#538 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:27,140 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/e45d81492a0949599caacfff19953b6e is 50, key is test_row_0/B:col10/1733708606254/Put/seqid=0 2024-12-09T01:43:27,143 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412099e26843500204e83b1f1cbd3f85e0e0b_45d234ddf430a4159b8bc5324edd88bc store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:27,145 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412099e26843500204e83b1f1cbd3f85e0e0b_45d234ddf430a4159b8bc5324edd88bc, store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:27,146 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099e26843500204e83b1f1cbd3f85e0e0b_45d234ddf430a4159b8bc5324edd88bc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:27,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742444_1620 (size=12104) 2024-12-09T01:43:27,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742445_1621 (size=4469) 2024-12-09T01:43:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-12-09T01:43:27,184 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:27,184 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:43:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-12-09T01:43:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T01:43:27,187 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:43:27,188 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:43:27,188 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:43:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T01:43:27,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-12-09T01:43:27,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:27,341 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 45d234ddf430a4159b8bc5324edd88bc 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-12-09T01:43:27,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=A 2024-12-09T01:43:27,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:27,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=B 2024-12-09T01:43:27,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:27,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=C 2024-12-09T01:43:27,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:27,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120986fe2fbaf87449a781e0d402f79912ed_45d234ddf430a4159b8bc5324edd88bc is 50, key is test_row_0/A:col10/1733708606318/Put/seqid=0 2024-12-09T01:43:27,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742446_1622 (size=12154) 2024-12-09T01:43:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:27,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:27,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708667466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:27,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708667469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T01:43:27,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708667498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:27,506 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:27,506 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,506 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708667498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:27,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708667498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:27,565 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/e45d81492a0949599caacfff19953b6e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/e45d81492a0949599caacfff19953b6e 2024-12-09T01:43:27,566 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#A#compaction#539 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:27,566 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/1f1568359e0f4f83a8c41b1d0997eb99 is 175, key is test_row_0/A:col10/1733708606254/Put/seqid=0 2024-12-09T01:43:27,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742447_1623 (size=31058) 2024-12-09T01:43:27,571 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/B of 45d234ddf430a4159b8bc5324edd88bc into e45d81492a0949599caacfff19953b6e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:27,571 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:27,571 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/B, priority=13, startTime=1733708607128; duration=0sec 2024-12-09T01:43:27,571 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:27,571 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:B 2024-12-09T01:43:27,571 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:27,572 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:27,572 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/C is initiating minor compaction (all files) 2024-12-09T01:43:27,572 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/C in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:27,573 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/a8c30e8d21564c51b22f8d56af71f732, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/94421a0b2a9044488603ce38c98e5fe1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/4a09a250d7a84389974b1b6d74090a1d] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=35.2 K 2024-12-09T01:43:27,578 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting a8c30e8d21564c51b22f8d56af71f732, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1733708605046 2024-12-09T01:43:27,578 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 94421a0b2a9044488603ce38c98e5fe1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1733708605073 2024-12-09T01:43:27,578 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a09a250d7a84389974b1b6d74090a1d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733708606243 2024-12-09T01:43:27,584 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#C#compaction#541 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:27,585 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/d80800658ddd486ba4e52213f1e98f08 is 50, key is test_row_0/C:col10/1733708606254/Put/seqid=0 2024-12-09T01:43:27,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742448_1624 (size=12104) 2024-12-09T01:43:27,591 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/d80800658ddd486ba4e52213f1e98f08 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/d80800658ddd486ba4e52213f1e98f08 2024-12-09T01:43:27,595 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/C of 45d234ddf430a4159b8bc5324edd88bc into d80800658ddd486ba4e52213f1e98f08(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:27,595 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:27,595 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/C, priority=13, startTime=1733708607129; duration=0sec 2024-12-09T01:43:27,595 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:27,595 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:C 2024-12-09T01:43:27,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708667603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:27,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708667603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:27,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:27,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708667613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:27,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:27,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:27,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708667614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:27,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708667614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,618 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:27,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:27,757 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120986fe2fbaf87449a781e0d402f79912ed_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120986fe2fbaf87449a781e0d402f79912ed_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:27,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/8d90d7a849e144c8a0bfb26ffa9d796b, store: [table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:27,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/8d90d7a849e144c8a0bfb26ffa9d796b is 175, key is test_row_0/A:col10/1733708606318/Put/seqid=0 2024-12-09T01:43:27,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742449_1625 (size=30955) 2024-12-09T01:43:27,762 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/8d90d7a849e144c8a0bfb26ffa9d796b 2024-12-09T01:43:27,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/5971f41b56ec4d27b180644be5c99b99 is 50, key is test_row_0/B:col10/1733708606318/Put/seqid=0 2024-12-09T01:43:27,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742450_1626 (size=12001) 2024-12-09T01:43:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T01:43:27,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708667813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:27,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708667813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:27,815 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:27,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,816 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:27,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708667824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:27,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:27,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708667824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:27,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:27,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:27,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708667824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:27,827 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:27,975 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/1f1568359e0f4f83a8c41b1d0997eb99 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/1f1568359e0f4f83a8c41b1d0997eb99 2024-12-09T01:43:27,979 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/A of 45d234ddf430a4159b8bc5324edd88bc into 1f1568359e0f4f83a8c41b1d0997eb99(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:27,979 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:27,979 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/A, priority=13, startTime=1733708607128; duration=0sec 2024-12-09T01:43:27,979 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:27,979 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:A 2024-12-09T01:43:28,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708668124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:28,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708668124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:28,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,129 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:28,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708668133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:28,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:28,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:28,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708668137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:28,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,137 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:28,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:28,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708668137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,138 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:28,175 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/5971f41b56ec4d27b180644be5c99b99 2024-12-09T01:43:28,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/147bb0c535a3417a814ec8e43848cdb5 is 50, key is test_row_0/C:col10/1733708606318/Put/seqid=0 2024-12-09T01:43:28,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742451_1627 (size=12001) 2024-12-09T01:43:28,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T01:43:28,588 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/147bb0c535a3417a814ec8e43848cdb5 2024-12-09T01:43:28,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/8d90d7a849e144c8a0bfb26ffa9d796b as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/8d90d7a849e144c8a0bfb26ffa9d796b 2024-12-09T01:43:28,596 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/8d90d7a849e144c8a0bfb26ffa9d796b, entries=150, sequenceid=81, filesize=30.2 K 2024-12-09T01:43:28,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/5971f41b56ec4d27b180644be5c99b99 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/5971f41b56ec4d27b180644be5c99b99 2024-12-09T01:43:28,600 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/5971f41b56ec4d27b180644be5c99b99, entries=150, sequenceid=81, filesize=11.7 K 2024-12-09T01:43:28,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/147bb0c535a3417a814ec8e43848cdb5 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/147bb0c535a3417a814ec8e43848cdb5 2024-12-09T01:43:28,605 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/147bb0c535a3417a814ec8e43848cdb5, entries=150, sequenceid=81, filesize=11.7 K 2024-12-09T01:43:28,606 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=53.67 KB/54960 for 45d234ddf430a4159b8bc5324edd88bc in 1265ms, sequenceid=81, compaction requested=false 2024-12-09T01:43:28,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:28,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:28,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-12-09T01:43:28,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-12-09T01:43:28,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-12-09T01:43:28,609 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4190 sec 2024-12-09T01:43:28,612 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.4250 sec 2024-12-09T01:43:28,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:28,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45d234ddf430a4159b8bc5324edd88bc 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T01:43:28,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=A 2024-12-09T01:43:28,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:28,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=B 2024-12-09T01:43:28,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:28,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=C 2024-12-09T01:43:28,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:28,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120938ec5ba4295a4c7cbf743e4a0874af3b_45d234ddf430a4159b8bc5324edd88bc is 50, key is test_row_0/A:col10/1733708608634/Put/seqid=0 2024-12-09T01:43:28,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742452_1628 (size=12154) 2024-12-09T01:43:28,645 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:28,649 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120938ec5ba4295a4c7cbf743e4a0874af3b_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120938ec5ba4295a4c7cbf743e4a0874af3b_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:28,650 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/2fb6b5e2e77c46859ae65103addd618e, store: [table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:28,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/2fb6b5e2e77c46859ae65103addd618e is 175, key is test_row_0/A:col10/1733708608634/Put/seqid=0 2024-12-09T01:43:28,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742453_1629 (size=30955) 2024-12-09T01:43:28,665 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:28,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708668661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:28,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708608656, max timestamp=1733708608663Num uniques 859; 2024-12-09T01:43:28,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708608656, max timestamp=1733708608663Num uniques 859; 2024-12-09T01:43:28,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:43:28,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:43:28,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1733708608656, max timestamp=1733708608663Num uniques 859; 2024-12-09T01:43:28,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ef6f18c58dc9:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-12-09T01:43:28,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:28,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708668684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:28,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:28,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708668685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:28,691 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:28,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,692 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:28,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:28,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708668691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:28,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708668690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:28,696 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,696 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,696 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:28,696 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:28,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708668773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:28,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:28,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708668793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:28,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:28,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708668794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:28,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:28,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:28,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708668804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:28,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708668804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:28,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708668983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:28,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:28,988 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:29,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:29,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708669004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:29,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:29,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:29,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708669004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:29,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:29,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:29,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708669013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:29,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:29,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708669013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:29,017 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:29,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,017 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:29,054 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/2fb6b5e2e77c46859ae65103addd618e 2024-12-09T01:43:29,068 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/01d423d3c44545ea8e6eeac41a1dabed is 50, key is test_row_0/B:col10/1733708608634/Put/seqid=0 2024-12-09T01:43:29,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742454_1630 (size=12001) 2024-12-09T01:43:29,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/01d423d3c44545ea8e6eeac41a1dabed 2024-12-09T01:43:29,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/15b2f7037b504aa69bfbfd6c9bbf9617 is 50, key is test_row_0/C:col10/1733708608634/Put/seqid=0 2024-12-09T01:43:29,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742455_1631 (size=12001) 2024-12-09T01:43:29,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708669294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:29,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:29,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708669314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:29,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:29,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708669314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:29,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:29,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:29,330 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:29,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708669323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:29,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708669323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,331 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:29,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-12-09T01:43:29,334 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:29,334 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:43:29,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-12-09T01:43:29,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T01:43:29,336 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:43:29,337 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:43:29,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:43:29,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T01:43:29,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/15b2f7037b504aa69bfbfd6c9bbf9617 2024-12-09T01:43:29,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/2fb6b5e2e77c46859ae65103addd618e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2fb6b5e2e77c46859ae65103addd618e 2024-12-09T01:43:29,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-09T01:43:29,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:29,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:29,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:29,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:29,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:29,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:29,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2fb6b5e2e77c46859ae65103addd618e, entries=150, sequenceid=96, filesize=30.2 K 2024-12-09T01:43:29,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/01d423d3c44545ea8e6eeac41a1dabed as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/01d423d3c44545ea8e6eeac41a1dabed 2024-12-09T01:43:29,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/01d423d3c44545ea8e6eeac41a1dabed, entries=150, sequenceid=96, filesize=11.7 K 2024-12-09T01:43:29,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/15b2f7037b504aa69bfbfd6c9bbf9617 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/15b2f7037b504aa69bfbfd6c9bbf9617 2024-12-09T01:43:29,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/15b2f7037b504aa69bfbfd6c9bbf9617, entries=150, sequenceid=96, filesize=11.7 K 2024-12-09T01:43:29,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 45d234ddf430a4159b8bc5324edd88bc in 864ms, sequenceid=96, compaction requested=true 2024-12-09T01:43:29,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:29,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:43:29,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:29,500 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:29,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:43:29,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:29,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:43:29,500 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:29,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:29,501 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:29,501 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:29,501 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/A is initiating minor compaction (all files) 2024-12-09T01:43:29,501 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/B is initiating minor compaction (all files) 2024-12-09T01:43:29,501 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/A in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:29,501 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/B in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:29,501 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/1f1568359e0f4f83a8c41b1d0997eb99, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/8d90d7a849e144c8a0bfb26ffa9d796b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2fb6b5e2e77c46859ae65103addd618e] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=90.8 K 2024-12-09T01:43:29,501 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/e45d81492a0949599caacfff19953b6e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/5971f41b56ec4d27b180644be5c99b99, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/01d423d3c44545ea8e6eeac41a1dabed] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=35.3 K 2024-12-09T01:43:29,501 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:29,501 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. files: [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/1f1568359e0f4f83a8c41b1d0997eb99, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/8d90d7a849e144c8a0bfb26ffa9d796b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2fb6b5e2e77c46859ae65103addd618e] 2024-12-09T01:43:29,501 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting e45d81492a0949599caacfff19953b6e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733708606243 2024-12-09T01:43:29,502 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f1568359e0f4f83a8c41b1d0997eb99, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733708606243 2024-12-09T01:43:29,502 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 5971f41b56ec4d27b180644be5c99b99, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733708606314 2024-12-09T01:43:29,502 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d90d7a849e144c8a0bfb26ffa9d796b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733708606314 2024-12-09T01:43:29,502 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 01d423d3c44545ea8e6eeac41a1dabed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733708607471 2024-12-09T01:43:29,502 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fb6b5e2e77c46859ae65103addd618e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733708607471 2024-12-09T01:43:29,507 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:29,509 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412094a04381a095549e9a771561e0a2e875f_45d234ddf430a4159b8bc5324edd88bc store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:29,509 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#B#compaction#547 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:29,509 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/cab8a3d005944004bc451a582519a5bb is 50, key is test_row_0/B:col10/1733708608634/Put/seqid=0 2024-12-09T01:43:29,510 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412094a04381a095549e9a771561e0a2e875f_45d234ddf430a4159b8bc5324edd88bc, store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:29,510 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412094a04381a095549e9a771561e0a2e875f_45d234ddf430a4159b8bc5324edd88bc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:29,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742456_1632 (size=4469) 2024-12-09T01:43:29,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742457_1633 (size=12207) 2024-12-09T01:43:29,522 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#A#compaction#548 average throughput is 1.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:29,523 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/92dd1c62bed643cf80d0137dd556d0e9 is 175, key is test_row_0/A:col10/1733708608634/Put/seqid=0 2024-12-09T01:43:29,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742458_1634 (size=31161) 2024-12-09T01:43:29,527 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/cab8a3d005944004bc451a582519a5bb as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/cab8a3d005944004bc451a582519a5bb 2024-12-09T01:43:29,532 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/B of 45d234ddf430a4159b8bc5324edd88bc into cab8a3d005944004bc451a582519a5bb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:29,532 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:29,532 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/B, priority=13, startTime=1733708609500; duration=0sec 2024-12-09T01:43:29,532 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:29,532 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:B 2024-12-09T01:43:29,533 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:29,534 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:29,534 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/C is initiating minor compaction (all files) 2024-12-09T01:43:29,534 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/C in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:29,534 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/d80800658ddd486ba4e52213f1e98f08, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/147bb0c535a3417a814ec8e43848cdb5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/15b2f7037b504aa69bfbfd6c9bbf9617] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=35.3 K 2024-12-09T01:43:29,534 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting d80800658ddd486ba4e52213f1e98f08, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=46, earliestPutTs=1733708606243 2024-12-09T01:43:29,534 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 147bb0c535a3417a814ec8e43848cdb5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1733708606314 2024-12-09T01:43:29,535 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 15b2f7037b504aa69bfbfd6c9bbf9617, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733708607471 2024-12-09T01:43:29,541 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#C#compaction#549 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:29,542 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/c2b1b091040f40b380741ce10e3d874f is 50, key is test_row_0/C:col10/1733708608634/Put/seqid=0 2024-12-09T01:43:29,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742459_1635 (size=12207) 2024-12-09T01:43:29,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-12-09T01:43:29,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:29,643 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 45d234ddf430a4159b8bc5324edd88bc 3/3 column families, dataSize=120.76 KB heapSize=335.77 KB 2024-12-09T01:43:29,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=A 2024-12-09T01:43:29,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:29,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=B 2024-12-09T01:43:29,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:29,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=C 2024-12-09T01:43:29,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:29,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f1ee0a1997e44b33be3ae016ae1fabe6_45d234ddf430a4159b8bc5324edd88bc is 50, key is test_row_0/A:col10/1733708608682/Put/seqid=0 2024-12-09T01:43:29,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T01:43:29,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742460_1636 (size=12154) 2024-12-09T01:43:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:29,807 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:29,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708669828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,836 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:29,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:29,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708669832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:29,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:29,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708669833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:29,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,839 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:29,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:29,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:29,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708669833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:29,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:29,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:29,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708669844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:29,845 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,845 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,845 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:29,932 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/92dd1c62bed643cf80d0137dd556d0e9 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/92dd1c62bed643cf80d0137dd556d0e9 2024-12-09T01:43:29,936 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/A of 45d234ddf430a4159b8bc5324edd88bc into 92dd1c62bed643cf80d0137dd556d0e9(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:29,936 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:29,936 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/A, priority=13, startTime=1733708609500; duration=0sec 2024-12-09T01:43:29,936 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:29,936 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:A 2024-12-09T01:43:29,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708669944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:29,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,947 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:29,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,947 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:29,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708669944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:29,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:29,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:29,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708669944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:29,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:29,948 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:29,961 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/c2b1b091040f40b380741ce10e3d874f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/c2b1b091040f40b380741ce10e3d874f 2024-12-09T01:43:29,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T01:43:29,965 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/C of 45d234ddf430a4159b8bc5324edd88bc into c2b1b091040f40b380741ce10e3d874f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:29,965 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:29,965 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/C, priority=13, startTime=1733708609500; duration=0sec 2024-12-09T01:43:29,965 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:29,965 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:C 2024-12-09T01:43:30,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:30,058 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f1ee0a1997e44b33be3ae016ae1fabe6_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f1ee0a1997e44b33be3ae016ae1fabe6_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:30,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/8d7e461a404d4cecbd7dafaaf8064c80, store: [table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:30,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/8d7e461a404d4cecbd7dafaaf8064c80 is 175, key is test_row_0/A:col10/1733708608682/Put/seqid=0 2024-12-09T01:43:30,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742461_1637 (size=30955) 2024-12-09T01:43:30,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:30,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708670154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:30,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708670154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:30,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:30,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708670154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:30,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:30,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:30,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708670463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:30,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:30,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708670463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:30,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:30,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:30,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:30,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:30,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:30,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:30,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:30,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708670463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:30,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:30,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:30,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:30,473 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=130, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/8d7e461a404d4cecbd7dafaaf8064c80 2024-12-09T01:43:30,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T01:43:30,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/3e7a93e86ce44cf595d6a5519779c922 is 50, key is test_row_0/B:col10/1733708608682/Put/seqid=0 2024-12-09T01:43:30,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742462_1638 (size=12001) 2024-12-09T01:43:30,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:30,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708670853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:30,858 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_0' in 45d234ddf430a4159b8bc5324edd88bc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:43:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:30,858 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:30,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:30,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708670864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:30,868 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_0' in 45d234ddf430a4159b8bc5324edd88bc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:43:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:30,869 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:30,884 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/3e7a93e86ce44cf595d6a5519779c922 2024-12-09T01:43:30,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/f2dda5cc00e849c98bd2f9b49b7873bd is 50, key is test_row_0/C:col10/1733708608682/Put/seqid=0 2024-12-09T01:43:30,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742463_1639 (size=12001) 2024-12-09T01:43:30,896 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/f2dda5cc00e849c98bd2f9b49b7873bd 2024-12-09T01:43:30,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/8d7e461a404d4cecbd7dafaaf8064c80 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/8d7e461a404d4cecbd7dafaaf8064c80 2024-12-09T01:43:30,903 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/8d7e461a404d4cecbd7dafaaf8064c80, entries=150, sequenceid=130, filesize=30.2 K 2024-12-09T01:43:30,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/3e7a93e86ce44cf595d6a5519779c922 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/3e7a93e86ce44cf595d6a5519779c922 2024-12-09T01:43:30,913 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/3e7a93e86ce44cf595d6a5519779c922, entries=150, sequenceid=130, filesize=11.7 K 2024-12-09T01:43:30,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/f2dda5cc00e849c98bd2f9b49b7873bd as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/f2dda5cc00e849c98bd2f9b49b7873bd 2024-12-09T01:43:30,917 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/f2dda5cc00e849c98bd2f9b49b7873bd, entries=150, sequenceid=130, filesize=11.7 K 2024-12-09T01:43:30,918 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~335.72 KB/343776, currentSize=60.38 KB/61830 for 45d234ddf430a4159b8bc5324edd88bc in 1275ms, sequenceid=130, compaction requested=false 2024-12-09T01:43:30,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:30,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:30,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-09T01:43:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-09T01:43:30,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-09T01:43:30,922 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5830 sec 2024-12-09T01:43:30,924 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.5890 sec 2024-12-09T01:43:30,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:30,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45d234ddf430a4159b8bc5324edd88bc 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T01:43:30,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=A 2024-12-09T01:43:30,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:30,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=B 2024-12-09T01:43:30,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:30,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=C 2024-12-09T01:43:30,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:30,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c3d0ade1a77c4578aefef4e8b1ef8f6a_45d234ddf430a4159b8bc5324edd88bc is 50, key is test_row_0/A:col10/1733708609831/Put/seqid=0 2024-12-09T01:43:31,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742464_1640 (size=17284) 2024-12-09T01:43:31,056 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:31,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708671051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:31,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708671054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:31,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:31,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708671055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:31,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,059 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:31,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:31,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708671164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:31,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708671164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:31,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:31,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,168 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:31,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708671164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:31,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:31,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708671373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:31,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708671373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:31,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:31,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708671373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,377 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,378 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:31,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:31,413 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:31,416 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c3d0ade1a77c4578aefef4e8b1ef8f6a_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c3d0ade1a77c4578aefef4e8b1ef8f6a_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:31,417 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/9ba75082de6f46a49010b565a00108f7, store: [table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:31,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/9ba75082de6f46a49010b565a00108f7 is 175, key is test_row_0/A:col10/1733708609831/Put/seqid=0 2024-12-09T01:43:31,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742465_1641 (size=48389) 2024-12-09T01:43:31,427 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=147, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/9ba75082de6f46a49010b565a00108f7 2024-12-09T01:43:31,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/21e57a310f824f17941956766ca96208 is 50, key is test_row_0/B:col10/1733708609831/Put/seqid=0 2024-12-09T01:43:31,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742466_1642 (size=12151) 2024-12-09T01:43:31,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/21e57a310f824f17941956766ca96208 2024-12-09T01:43:31,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/718227b39ad34a0888c95cbfbc878694 is 50, key is test_row_0/C:col10/1733708609831/Put/seqid=0 2024-12-09T01:43:31,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742467_1643 (size=12151) 2024-12-09T01:43:31,466 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=147 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/718227b39ad34a0888c95cbfbc878694 2024-12-09T01:43:31,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/9ba75082de6f46a49010b565a00108f7 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/9ba75082de6f46a49010b565a00108f7 2024-12-09T01:43:31,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/9ba75082de6f46a49010b565a00108f7, entries=250, sequenceid=147, filesize=47.3 K 2024-12-09T01:43:31,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/21e57a310f824f17941956766ca96208 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/21e57a310f824f17941956766ca96208 2024-12-09T01:43:31,481 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/21e57a310f824f17941956766ca96208, entries=150, sequenceid=147, filesize=11.9 K 2024-12-09T01:43:31,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/718227b39ad34a0888c95cbfbc878694 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/718227b39ad34a0888c95cbfbc878694 2024-12-09T01:43:31,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/718227b39ad34a0888c95cbfbc878694, entries=150, sequenceid=147, filesize=11.9 K 2024-12-09T01:43:31,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 45d234ddf430a4159b8bc5324edd88bc in 510ms, sequenceid=147, compaction requested=true 2024-12-09T01:43:31,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:31,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:43:31,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:31,488 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:31,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:43:31,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:31,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:43:31,488 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:31,488 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:31,489 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:31,489 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:31,489 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/A is initiating minor compaction (all files) 2024-12-09T01:43:31,489 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/B is initiating minor compaction (all files) 2024-12-09T01:43:31,489 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/A in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:31,489 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/B in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:31,489 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/92dd1c62bed643cf80d0137dd556d0e9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/8d7e461a404d4cecbd7dafaaf8064c80, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/9ba75082de6f46a49010b565a00108f7] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=107.9 K 2024-12-09T01:43:31,489 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/cab8a3d005944004bc451a582519a5bb, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/3e7a93e86ce44cf595d6a5519779c922, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/21e57a310f824f17941956766ca96208] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=35.5 K 2024-12-09T01:43:31,489 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:31,489 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. files: [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/92dd1c62bed643cf80d0137dd556d0e9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/8d7e461a404d4cecbd7dafaaf8064c80, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/9ba75082de6f46a49010b565a00108f7] 2024-12-09T01:43:31,490 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92dd1c62bed643cf80d0137dd556d0e9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733708607471 2024-12-09T01:43:31,490 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting cab8a3d005944004bc451a582519a5bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733708607471 2024-12-09T01:43:31,490 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d7e461a404d4cecbd7dafaaf8064c80, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733708608680 2024-12-09T01:43:31,490 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e7a93e86ce44cf595d6a5519779c922, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733708608680 2024-12-09T01:43:31,490 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ba75082de6f46a49010b565a00108f7, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733708609827 2024-12-09T01:43:31,491 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 21e57a310f824f17941956766ca96208, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733708609831 2024-12-09T01:43:31,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-09T01:43:31,494 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:31,494 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:43:31,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-12-09T01:43:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T01:43:31,496 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:43:31,497 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:43:31,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:43:31,500 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:31,504 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#B#compaction#557 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:31,504 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120951e989e4a9ec45b5a6db0693e815f109_45d234ddf430a4159b8bc5324edd88bc store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:31,504 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/ee268eee066540f3b012585022f53942 is 50, key is test_row_0/B:col10/1733708609831/Put/seqid=0 2024-12-09T01:43:31,507 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120951e989e4a9ec45b5a6db0693e815f109_45d234ddf430a4159b8bc5324edd88bc, store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:31,507 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120951e989e4a9ec45b5a6db0693e815f109_45d234ddf430a4159b8bc5324edd88bc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:31,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742469_1645 (size=4469) 2024-12-09T01:43:31,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742468_1644 (size=12459) 2024-12-09T01:43:31,513 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#A#compaction#556 average throughput is 1.88 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:31,513 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/c7dbcac6f2c24434a185b202767700b9 is 175, key is test_row_0/A:col10/1733708609831/Put/seqid=0 2024-12-09T01:43:31,517 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/ee268eee066540f3b012585022f53942 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/ee268eee066540f3b012585022f53942 2024-12-09T01:43:31,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742470_1646 (size=31413) 2024-12-09T01:43:31,522 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/B of 45d234ddf430a4159b8bc5324edd88bc into ee268eee066540f3b012585022f53942(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:31,522 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:31,522 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/B, priority=13, startTime=1733708611488; duration=0sec 2024-12-09T01:43:31,522 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:31,522 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:B 2024-12-09T01:43:31,522 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:31,524 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:31,524 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/C is initiating minor compaction (all files) 2024-12-09T01:43:31,524 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/C in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:31,524 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/c2b1b091040f40b380741ce10e3d874f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/f2dda5cc00e849c98bd2f9b49b7873bd, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/718227b39ad34a0888c95cbfbc878694] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=35.5 K 2024-12-09T01:43:31,525 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting c2b1b091040f40b380741ce10e3d874f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1733708607471 2024-12-09T01:43:31,525 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting f2dda5cc00e849c98bd2f9b49b7873bd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1733708608680 2024-12-09T01:43:31,526 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 718227b39ad34a0888c95cbfbc878694, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733708609831 2024-12-09T01:43:31,529 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/c7dbcac6f2c24434a185b202767700b9 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/c7dbcac6f2c24434a185b202767700b9 2024-12-09T01:43:31,535 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#C#compaction#558 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:31,536 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/65fcfba4b5f74263afa1fc0aee829f1e is 50, key is test_row_0/C:col10/1733708609831/Put/seqid=0 2024-12-09T01:43:31,537 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/A of 45d234ddf430a4159b8bc5324edd88bc into c7dbcac6f2c24434a185b202767700b9(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:31,537 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:31,537 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/A, priority=13, startTime=1733708611487; duration=0sec 2024-12-09T01:43:31,538 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:31,538 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:A 2024-12-09T01:43:31,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742471_1647 (size=12459) 2024-12-09T01:43:31,544 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/65fcfba4b5f74263afa1fc0aee829f1e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/65fcfba4b5f74263afa1fc0aee829f1e 2024-12-09T01:43:31,549 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/C of 45d234ddf430a4159b8bc5324edd88bc into 65fcfba4b5f74263afa1fc0aee829f1e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:31,549 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:31,549 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/C, priority=13, startTime=1733708611488; duration=0sec 2024-12-09T01:43:31,549 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:31,549 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:C 2024-12-09T01:43:31,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T01:43:31,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-12-09T01:43:31,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:31,649 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 45d234ddf430a4159b8bc5324edd88bc 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-09T01:43:31,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=A 2024-12-09T01:43:31,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:31,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=B 2024-12-09T01:43:31,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:31,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=C 2024-12-09T01:43:31,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:31,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096514b97ac60e4274b4073ecb92ba4904_45d234ddf430a4159b8bc5324edd88bc is 50, key is test_row_0/A:col10/1733708611053/Put/seqid=0 2024-12-09T01:43:31,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742472_1648 (size=12304) 2024-12-09T01:43:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:31,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:31,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:31,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708671726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:31,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:31,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708671727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:31,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,732 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:31,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,732 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:31,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:31,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708671731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,735 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:31,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T01:43:31,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:31,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708671843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:31,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708671843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:31,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:31,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708671843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:31,848 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:31,849 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:32,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708672053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:32,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:32,057 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708672053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:32,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:32,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708672053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:32,058 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:32,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:32,064 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096514b97ac60e4274b4073ecb92ba4904_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096514b97ac60e4274b4073ecb92ba4904_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:32,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/0dc51fe951b04e8fbcabe61133271f35, store: [table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:32,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/0dc51fe951b04e8fbcabe61133271f35 is 175, key is test_row_0/A:col10/1733708611053/Put/seqid=0 2024-12-09T01:43:32,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742473_1649 (size=31105) 2024-12-09T01:43:32,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T01:43:32,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708672364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:32,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:32,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708672364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:32,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:32,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:32,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708672364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:32,368 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:32,470 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/0dc51fe951b04e8fbcabe61133271f35 2024-12-09T01:43:32,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/c196e3bdcd3a4181870eb762ef539c7c is 50, key is test_row_0/B:col10/1733708611053/Put/seqid=0 2024-12-09T01:43:32,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742474_1650 (size=12151) 2024-12-09T01:43:32,482 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/c196e3bdcd3a4181870eb762ef539c7c 2024-12-09T01:43:32,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/6b6e0a4f22ee4424acfb3c2de74725be is 50, key is test_row_0/C:col10/1733708611053/Put/seqid=0 2024-12-09T01:43:32,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742475_1651 (size=12151) 2024-12-09T01:43:32,494 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/6b6e0a4f22ee4424acfb3c2de74725be 2024-12-09T01:43:32,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/0dc51fe951b04e8fbcabe61133271f35 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/0dc51fe951b04e8fbcabe61133271f35 2024-12-09T01:43:32,504 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/0dc51fe951b04e8fbcabe61133271f35, entries=150, sequenceid=167, filesize=30.4 K 2024-12-09T01:43:32,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/c196e3bdcd3a4181870eb762ef539c7c as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/c196e3bdcd3a4181870eb762ef539c7c 2024-12-09T01:43:32,509 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/c196e3bdcd3a4181870eb762ef539c7c, entries=150, sequenceid=167, filesize=11.9 K 2024-12-09T01:43:32,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/6b6e0a4f22ee4424acfb3c2de74725be as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/6b6e0a4f22ee4424acfb3c2de74725be 2024-12-09T01:43:32,513 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/6b6e0a4f22ee4424acfb3c2de74725be, entries=150, sequenceid=167, filesize=11.9 K 2024-12-09T01:43:32,514 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 45d234ddf430a4159b8bc5324edd88bc in 865ms, sequenceid=167, compaction requested=false 2024-12-09T01:43:32,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:32,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:32,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-09T01:43:32,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-12-09T01:43:32,517 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-12-09T01:43:32,517 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0190 sec 2024-12-09T01:43:32,519 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.0240 sec 2024-12-09T01:43:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-09T01:43:32,633 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:32,634 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:43:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-12-09T01:43:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T01:43:32,635 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:43:32,636 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:43:32,636 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:43:32,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T01:43:32,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-12-09T01:43:32,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:32,788 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 45d234ddf430a4159b8bc5324edd88bc 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T01:43:32,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=A 2024-12-09T01:43:32,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:32,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=B 2024-12-09T01:43:32,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:32,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=C 2024-12-09T01:43:32,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:32,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092fdf8b1b10184d1990f3511a025bb381_45d234ddf430a4159b8bc5324edd88bc is 50, key is test_row_0/A:col10/1733708611711/Put/seqid=0 2024-12-09T01:43:32,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742476_1652 (size=12304) 2024-12-09T01:43:32,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:32,812 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092fdf8b1b10184d1990f3511a025bb381_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092fdf8b1b10184d1990f3511a025bb381_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:32,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/56a881e3565843bc8b85b94335b10b80, store: [table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:32,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/56a881e3565843bc8b85b94335b10b80 is 175, key is test_row_0/A:col10/1733708611711/Put/seqid=0 2024-12-09T01:43:32,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742477_1653 (size=31105) 2024-12-09T01:43:32,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:32,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:32,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708672891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:32,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:32,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:32,895 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:32,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:32,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708672893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:32,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:32,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708672893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:32,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:32,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708672894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:32,899 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_0' in 45d234ddf430a4159b8bc5324edd88bc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:43:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:32,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:32,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:32,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708672895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:32,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:32,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T01:43:33,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708673003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:33,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708673003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:33,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708673003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:33,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708673004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,009 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:33,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:33,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708673213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:33,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:33,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708673213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,216 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,216 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:33,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708673213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:33,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708673214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:33,217 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/56a881e3565843bc8b85b94335b10b80 2024-12-09T01:43:33,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/9b3a8742fc6f4509b76f60c5fc52cd9f is 50, key is test_row_0/B:col10/1733708611711/Put/seqid=0 2024-12-09T01:43:33,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742478_1654 (size=12151) 2024-12-09T01:43:33,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T01:43:33,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:33,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708673523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:33,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708673523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:33,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708673523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:33,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708673524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:33,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,529 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:33,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:33,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:33,629 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/9b3a8742fc6f4509b76f60c5fc52cd9f 2024-12-09T01:43:33,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/07cd7f8361514cae81b86f10240726b9 is 50, key is test_row_0/C:col10/1733708611711/Put/seqid=0 2024-12-09T01:43:33,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742479_1655 (size=12151) 2024-12-09T01:43:33,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T01:43:34,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:34,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708674034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:34,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708674034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:34,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:34,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708674034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:34,036 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:34,039 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/07cd7f8361514cae81b86f10240726b9 2024-12-09T01:43:34,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/56a881e3565843bc8b85b94335b10b80 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/56a881e3565843bc8b85b94335b10b80 2024-12-09T01:43:34,046 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/56a881e3565843bc8b85b94335b10b80, entries=150, sequenceid=182, filesize=30.4 K 2024-12-09T01:43:34,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:34,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708674043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:34,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:34,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:34,047 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:34,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/9b3a8742fc6f4509b76f60c5fc52cd9f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/9b3a8742fc6f4509b76f60c5fc52cd9f 2024-12-09T01:43:34,051 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/9b3a8742fc6f4509b76f60c5fc52cd9f, entries=150, sequenceid=182, filesize=11.9 K 2024-12-09T01:43:34,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/07cd7f8361514cae81b86f10240726b9 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/07cd7f8361514cae81b86f10240726b9 2024-12-09T01:43:34,054 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/07cd7f8361514cae81b86f10240726b9, entries=150, sequenceid=182, filesize=11.9 K 2024-12-09T01:43:34,055 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 45d234ddf430a4159b8bc5324edd88bc in 1267ms, sequenceid=182, compaction requested=true 2024-12-09T01:43:34,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:34,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:34,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-12-09T01:43:34,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-12-09T01:43:34,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-12-09T01:43:34,058 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4210 sec 2024-12-09T01:43:34,060 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.4250 sec 2024-12-09T01:43:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-12-09T01:43:34,784 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:34,784 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:43:34,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-12-09T01:43:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T01:43:34,786 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:43:34,787 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:43:34,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:43:34,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T01:43:34,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-12-09T01:43:34,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:34,940 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 45d234ddf430a4159b8bc5324edd88bc 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-09T01:43:34,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=A 2024-12-09T01:43:34,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:34,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=B 2024-12-09T01:43:34,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:34,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=C 2024-12-09T01:43:34,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:34,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092f0d0fbebdab443eb83637909a4ec49f_45d234ddf430a4159b8bc5324edd88bc is 50, key is test_row_0/A:col10/1733708612892/Put/seqid=0 2024-12-09T01:43:34,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742480_1656 (size=12304) 2024-12-09T01:43:35,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:35,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:35,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708675094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:35,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:35,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:35,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708675094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:35,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,102 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:35,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708675094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:35,103 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708675095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:35,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T01:43:35,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708675213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:35,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708675214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,217 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:35,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708675214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:35,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:35,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708675214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:35,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:35,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:35,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:35,377 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412092f0d0fbebdab443eb83637909a4ec49f_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092f0d0fbebdab443eb83637909a4ec49f_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:35,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/e32d8cccd1854c5597a40acf0a6f7bf5, store: [table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:35,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/e32d8cccd1854c5597a40acf0a6f7bf5 is 175, key is test_row_0/A:col10/1733708612892/Put/seqid=0 2024-12-09T01:43:35,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742481_1657 (size=31105) 2024-12-09T01:43:35,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T01:43:35,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708675424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:35,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708675424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:35,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,426 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:35,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:35,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708675424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:35,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:35,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:35,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708675424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:35,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:35,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:35,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:35,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708675733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:35,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,737 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708675734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:35,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708675734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:35,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708675734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:35,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:35,783 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=199, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/e32d8cccd1854c5597a40acf0a6f7bf5 2024-12-09T01:43:35,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/31d8d9ce9ecc4034b0bdefe51cb8955a is 50, key is test_row_0/B:col10/1733708612892/Put/seqid=0 2024-12-09T01:43:35,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742482_1658 (size=12151) 2024-12-09T01:43:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T01:43:36,194 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/31d8d9ce9ecc4034b0bdefe51cb8955a 2024-12-09T01:43:36,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/49ab6a902153446facec31c8ecd79c29 is 50, key is test_row_0/C:col10/1733708612892/Put/seqid=0 2024-12-09T01:43:36,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742483_1659 (size=12151) 2024-12-09T01:43:36,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:36,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708676244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:36,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:36,246 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:36,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:36,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708676244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:36,247 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:36,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:36,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:36,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:36,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:36,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708676244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:36,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:36,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:36,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:36,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:36,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708676253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:36,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:36,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:36,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:36,606 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/49ab6a902153446facec31c8ecd79c29 2024-12-09T01:43:36,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/e32d8cccd1854c5597a40acf0a6f7bf5 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/e32d8cccd1854c5597a40acf0a6f7bf5 2024-12-09T01:43:36,619 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/e32d8cccd1854c5597a40acf0a6f7bf5, entries=150, sequenceid=199, filesize=30.4 K 2024-12-09T01:43:36,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/31d8d9ce9ecc4034b0bdefe51cb8955a as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/31d8d9ce9ecc4034b0bdefe51cb8955a 2024-12-09T01:43:36,623 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/31d8d9ce9ecc4034b0bdefe51cb8955a, entries=150, sequenceid=199, filesize=11.9 K 2024-12-09T01:43:36,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/49ab6a902153446facec31c8ecd79c29 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/49ab6a902153446facec31c8ecd79c29 2024-12-09T01:43:36,627 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/49ab6a902153446facec31c8ecd79c29, entries=150, sequenceid=199, filesize=11.9 K 2024-12-09T01:43:36,628 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 45d234ddf430a4159b8bc5324edd88bc in 1688ms, sequenceid=199, compaction requested=true 2024-12-09T01:43:36,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:36,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:36,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-12-09T01:43:36,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-12-09T01:43:36,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-12-09T01:43:36,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8420 sec 2024-12-09T01:43:36,632 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.8470 sec 2024-12-09T01:43:36,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:36,917 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45d234ddf430a4159b8bc5324edd88bc 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-12-09T01:43:36,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=A 2024-12-09T01:43:36,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:36,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=B 2024-12-09T01:43:36,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:36,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=C 2024-12-09T01:43:36,917 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:36,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096891cfb63c1d4409a78544c3eec481a0_45d234ddf430a4159b8bc5324edd88bc is 50, key is test_row_0/A:col10/1733708616915/Put/seqid=0 2024-12-09T01:43:36,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742484_1660 (size=14794) 2024-12-09T01:43:36,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-09T01:43:36,934 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:36,934 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:43:36,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-12-09T01:43:36,935 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:43:36,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T01:43:36,936 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:43:36,936 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:43:37,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:37,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708676998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:37,005 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:37,005 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:37,005 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:37,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T01:43:37,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-09T01:43:37,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:37,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:37,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:37,088 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:37,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:37,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:37,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708677113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:37,114 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:37,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:37,115 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:37,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-09T01:43:37,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:37,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:37,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:37,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:37,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:37,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:37,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T01:43:37,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708677253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:37,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:37,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708677253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:37,258 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_1' in 45d234ddf430a4159b8bc5324edd88bc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:43:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:37,258 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_1' in 45d234ddf430a4159b8bc5324edd88bc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:43:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:37,259 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:37,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:37,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708677264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:37,266 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_2' in 45d234ddf430a4159b8bc5324edd88bc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:43:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:37,267 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:37,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708677273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:37,277 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_2' in 45d234ddf430a4159b8bc5324edd88bc of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:43:37,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:37,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:37,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:37,327 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:37,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:37,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708677324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:37,329 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:37,331 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412096891cfb63c1d4409a78544c3eec481a0_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096891cfb63c1d4409a78544c3eec481a0_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:37,332 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/3a4a95b7da0e41e3b31a98358b050121, store: [table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:37,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/3a4a95b7da0e41e3b31a98358b050121 is 175, key is test_row_0/A:col10/1733708616915/Put/seqid=0 2024-12-09T01:43:37,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742485_1661 (size=39749) 2024-12-09T01:43:37,337 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/3a4a95b7da0e41e3b31a98358b050121 2024-12-09T01:43:37,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/11874b8cdf804614a06ded2390f1cd44 is 50, key is test_row_0/B:col10/1733708616915/Put/seqid=0 2024-12-09T01:43:37,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742486_1662 (size=12151) 2024-12-09T01:43:37,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-09T01:43:37,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:37,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:37,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:37,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:37,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:37,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:37,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-09T01:43:37,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:37,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:37,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:37,547 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:37,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:37,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:37,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T01:43:37,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:37,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708677633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:37,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:37,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-09T01:43:37,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:37,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:37,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:37,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:37,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:37,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:37,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/11874b8cdf804614a06ded2390f1cd44 2024-12-09T01:43:37,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/1f0e0601be6a4d218740549705dc5994 is 50, key is test_row_0/C:col10/1733708616915/Put/seqid=0 2024-12-09T01:43:37,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742487_1663 (size=12151) 2024-12-09T01:43:37,767 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/1f0e0601be6a4d218740549705dc5994 2024-12-09T01:43:37,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/3a4a95b7da0e41e3b31a98358b050121 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/3a4a95b7da0e41e3b31a98358b050121 2024-12-09T01:43:37,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/3a4a95b7da0e41e3b31a98358b050121, entries=200, sequenceid=216, filesize=38.8 K 2024-12-09T01:43:37,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/11874b8cdf804614a06ded2390f1cd44 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/11874b8cdf804614a06ded2390f1cd44 2024-12-09T01:43:37,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/11874b8cdf804614a06ded2390f1cd44, entries=150, sequenceid=216, filesize=11.9 K 2024-12-09T01:43:37,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/1f0e0601be6a4d218740549705dc5994 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/1f0e0601be6a4d218740549705dc5994 2024-12-09T01:43:37,784 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/1f0e0601be6a4d218740549705dc5994, entries=150, sequenceid=216, filesize=11.9 K 2024-12-09T01:43:37,786 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 45d234ddf430a4159b8bc5324edd88bc in 870ms, sequenceid=216, compaction requested=true 2024-12-09T01:43:37,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:37,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:43:37,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:37,786 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T01:43:37,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:43:37,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:37,786 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T01:43:37,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:43:37,786 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:37,787 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 164477 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T01:43:37,787 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/A is initiating minor compaction (all files) 2024-12-09T01:43:37,787 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/A in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:37,787 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/c7dbcac6f2c24434a185b202767700b9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/0dc51fe951b04e8fbcabe61133271f35, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/56a881e3565843bc8b85b94335b10b80, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/e32d8cccd1854c5597a40acf0a6f7bf5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/3a4a95b7da0e41e3b31a98358b050121] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=160.6 K 2024-12-09T01:43:37,787 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:37,787 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. files: [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/c7dbcac6f2c24434a185b202767700b9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/0dc51fe951b04e8fbcabe61133271f35, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/56a881e3565843bc8b85b94335b10b80, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/e32d8cccd1854c5597a40acf0a6f7bf5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/3a4a95b7da0e41e3b31a98358b050121] 2024-12-09T01:43:37,788 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61063 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T01:43:37,788 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7dbcac6f2c24434a185b202767700b9, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733708609831 2024-12-09T01:43:37,788 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/B is initiating minor compaction (all files) 2024-12-09T01:43:37,788 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/B in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:37,788 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/ee268eee066540f3b012585022f53942, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/c196e3bdcd3a4181870eb762ef539c7c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/9b3a8742fc6f4509b76f60c5fc52cd9f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/31d8d9ce9ecc4034b0bdefe51cb8955a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/11874b8cdf804614a06ded2390f1cd44] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=59.6 K 2024-12-09T01:43:37,788 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting ee268eee066540f3b012585022f53942, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733708609831 2024-12-09T01:43:37,788 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dc51fe951b04e8fbcabe61133271f35, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733708611047 2024-12-09T01:43:37,789 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting c196e3bdcd3a4181870eb762ef539c7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733708611047 2024-12-09T01:43:37,789 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b3a8742fc6f4509b76f60c5fc52cd9f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733708611711 2024-12-09T01:43:37,789 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56a881e3565843bc8b85b94335b10b80, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733708611711 2024-12-09T01:43:37,789 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 31d8d9ce9ecc4034b0bdefe51cb8955a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733708612889 2024-12-09T01:43:37,789 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting e32d8cccd1854c5597a40acf0a6f7bf5, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733708612889 2024-12-09T01:43:37,789 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 11874b8cdf804614a06ded2390f1cd44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733708615060 2024-12-09T01:43:37,790 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a4a95b7da0e41e3b31a98358b050121, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733708615060 2024-12-09T01:43:37,802 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#B#compaction#571 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:37,802 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/6663fa7bd87d48e99a28f9c4b8a0e426 is 50, key is test_row_0/B:col10/1733708616915/Put/seqid=0 2024-12-09T01:43:37,804 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:37,805 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209def5460c4589474bac2ef39ef89784c6_45d234ddf430a4159b8bc5324edd88bc store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:37,808 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209def5460c4589474bac2ef39ef89784c6_45d234ddf430a4159b8bc5324edd88bc, store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:37,808 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209def5460c4589474bac2ef39ef89784c6_45d234ddf430a4159b8bc5324edd88bc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:37,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742488_1664 (size=12629) 2024-12-09T01:43:37,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742489_1665 (size=4469) 2024-12-09T01:43:37,831 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#A#compaction#572 average throughput is 0.90 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:37,832 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/06e15ecea86343fbab386b1659f9d69c is 175, key is test_row_0/A:col10/1733708616915/Put/seqid=0 2024-12-09T01:43:37,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742490_1666 (size=31583) 2024-12-09T01:43:37,851 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/06e15ecea86343fbab386b1659f9d69c as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/06e15ecea86343fbab386b1659f9d69c 2024-12-09T01:43:37,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-12-09T01:43:37,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:37,854 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 45d234ddf430a4159b8bc5324edd88bc 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-12-09T01:43:37,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=A 2024-12-09T01:43:37,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:37,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=B 2024-12-09T01:43:37,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:37,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=C 2024-12-09T01:43:37,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:37,857 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/A of 45d234ddf430a4159b8bc5324edd88bc into 06e15ecea86343fbab386b1659f9d69c(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:37,857 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:37,857 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/A, priority=11, startTime=1733708617786; duration=0sec 2024-12-09T01:43:37,857 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:37,857 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:A 2024-12-09T01:43:37,857 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-12-09T01:43:37,860 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61063 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-12-09T01:43:37,860 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/C is initiating minor compaction (all files) 2024-12-09T01:43:37,860 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/C in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:37,860 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/65fcfba4b5f74263afa1fc0aee829f1e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/6b6e0a4f22ee4424acfb3c2de74725be, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/07cd7f8361514cae81b86f10240726b9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/49ab6a902153446facec31c8ecd79c29, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/1f0e0601be6a4d218740549705dc5994] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=59.6 K 2024-12-09T01:43:37,860 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65fcfba4b5f74263afa1fc0aee829f1e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=147, earliestPutTs=1733708609831 2024-12-09T01:43:37,861 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b6e0a4f22ee4424acfb3c2de74725be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1733708611047 2024-12-09T01:43:37,861 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07cd7f8361514cae81b86f10240726b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1733708611711 2024-12-09T01:43:37,862 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49ab6a902153446facec31c8ecd79c29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1733708612889 2024-12-09T01:43:37,862 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f0e0601be6a4d218740549705dc5994, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733708615060 2024-12-09T01:43:37,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099adc93a925dc43eaaafafa25b78497a4_45d234ddf430a4159b8bc5324edd88bc is 50, key is test_row_0/A:col10/1733708616932/Put/seqid=0 2024-12-09T01:43:37,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742491_1667 (size=12304) 2024-12-09T01:43:37,878 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#C#compaction#574 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:37,879 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/bf9c40a57eba4756bc6136a4dd2c1792 is 50, key is test_row_0/C:col10/1733708616915/Put/seqid=0 2024-12-09T01:43:37,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742492_1668 (size=12629) 2024-12-09T01:43:37,888 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/bf9c40a57eba4756bc6136a4dd2c1792 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/bf9c40a57eba4756bc6136a4dd2c1792 2024-12-09T01:43:37,892 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/C of 45d234ddf430a4159b8bc5324edd88bc into bf9c40a57eba4756bc6136a4dd2c1792(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:37,892 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:37,892 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/C, priority=11, startTime=1733708617786; duration=0sec 2024-12-09T01:43:37,892 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:37,892 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:C 2024-12-09T01:43:38,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T01:43:38,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:38,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:38,215 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/6663fa7bd87d48e99a28f9c4b8a0e426 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/6663fa7bd87d48e99a28f9c4b8a0e426 2024-12-09T01:43:38,219 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/B of 45d234ddf430a4159b8bc5324edd88bc into 6663fa7bd87d48e99a28f9c4b8a0e426(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:38,219 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:38,219 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/B, priority=11, startTime=1733708617786; duration=0sec 2024-12-09T01:43:38,219 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:38,219 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:B 2024-12-09T01:43:38,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:38,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708678231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:38,236 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:38,236 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:38,236 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:38,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:38,278 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099adc93a925dc43eaaafafa25b78497a4_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099adc93a925dc43eaaafafa25b78497a4_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:38,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/2918bb26cd104435acdea2a6a24d1e99, store: [table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:38,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/2918bb26cd104435acdea2a6a24d1e99 is 175, key is test_row_0/A:col10/1733708616932/Put/seqid=0 2024-12-09T01:43:38,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742493_1669 (size=31105) 2024-12-09T01:43:38,282 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=231, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/2918bb26cd104435acdea2a6a24d1e99 2024-12-09T01:43:38,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/0c549fa6844144b1a84619c254f9d8af is 50, key is test_row_0/B:col10/1733708616932/Put/seqid=0 2024-12-09T01:43:38,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742494_1670 (size=12151) 2024-12-09T01:43:38,302 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/0c549fa6844144b1a84619c254f9d8af 2024-12-09T01:43:38,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/f55df748e4f14cc5beaadd668cf55904 is 50, key is test_row_0/C:col10/1733708616932/Put/seqid=0 2024-12-09T01:43:38,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742495_1671 (size=12151) 2024-12-09T01:43:38,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:38,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708678343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:38,346 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:38,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708678553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:38,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:38,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:38,556 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:38,711 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/f55df748e4f14cc5beaadd668cf55904 2024-12-09T01:43:38,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/2918bb26cd104435acdea2a6a24d1e99 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2918bb26cd104435acdea2a6a24d1e99 2024-12-09T01:43:38,719 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2918bb26cd104435acdea2a6a24d1e99, entries=150, sequenceid=231, filesize=30.4 K 2024-12-09T01:43:38,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/0c549fa6844144b1a84619c254f9d8af as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/0c549fa6844144b1a84619c254f9d8af 2024-12-09T01:43:38,722 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/0c549fa6844144b1a84619c254f9d8af, entries=150, sequenceid=231, filesize=11.9 K 2024-12-09T01:43:38,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/f55df748e4f14cc5beaadd668cf55904 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/f55df748e4f14cc5beaadd668cf55904 2024-12-09T01:43:38,726 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/f55df748e4f14cc5beaadd668cf55904, entries=150, sequenceid=231, filesize=11.9 K 2024-12-09T01:43:38,727 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 45d234ddf430a4159b8bc5324edd88bc in 873ms, sequenceid=231, compaction requested=false 2024-12-09T01:43:38,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:38,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:38,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-12-09T01:43:38,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-12-09T01:43:38,731 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-12-09T01:43:38,731 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7930 sec 2024-12-09T01:43:38,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.7980 sec 2024-12-09T01:43:38,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:38,867 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45d234ddf430a4159b8bc5324edd88bc 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-12-09T01:43:38,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=A 2024-12-09T01:43:38,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:38,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=B 2024-12-09T01:43:38,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:38,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=C 2024-12-09T01:43:38,867 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:38,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209831e6b185fb84289ae037cf98a7ed130_45d234ddf430a4159b8bc5324edd88bc is 50, key is test_row_0/A:col10/1733708618865/Put/seqid=0 2024-12-09T01:43:38,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742496_1672 (size=14794) 2024-12-09T01:43:38,879 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:38,882 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209831e6b185fb84289ae037cf98a7ed130_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209831e6b185fb84289ae037cf98a7ed130_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:38,883 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/693c46b1840a4f2893ca4b633a221d7e, store: [table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:38,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/693c46b1840a4f2893ca4b633a221d7e is 175, key is test_row_0/A:col10/1733708618865/Put/seqid=0 2024-12-09T01:43:38,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742497_1673 (size=39749) 2024-12-09T01:43:38,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:38,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708678924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:38,928 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:39,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:39,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708679034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:39,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-12-09T01:43:39,084 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:39,084 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:43:39,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-12-09T01:43:39,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-09T01:43:39,086 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:43:39,087 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:43:39,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:43:39,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-09T01:43:39,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-09T01:43:39,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:39,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:39,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:39,240 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:39,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:39,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:39,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708679244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:39,245 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:39,245 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:39,245 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:39,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708679283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:39,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708679283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:39,289 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_1' in 45d234ddf430a4159b8bc5324edd88bc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:43:39,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:39,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:39,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708679283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:39,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:39,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:39,289 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_1' in 45d234ddf430a4159b8bc5324edd88bc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:43:39,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:39,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:39,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:39,290 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_2' in 45d234ddf430a4159b8bc5324edd88bc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:43:39,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:39,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:39,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:39,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708679294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:39,296 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_2' in 45d234ddf430a4159b8bc5324edd88bc of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:43:39,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:39,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:39,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:39,299 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/693c46b1840a4f2893ca4b633a221d7e 2024-12-09T01:43:39,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/9d03af4f58d944cc8e062b3e3e414de1 is 50, key is test_row_0/B:col10/1733708618865/Put/seqid=0 2024-12-09T01:43:39,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742498_1674 (size=12151) 2024-12-09T01:43:39,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/9d03af4f58d944cc8e062b3e3e414de1 2024-12-09T01:43:39,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/416b71c4f039476f98cc0493575ac6d5 is 50, key is test_row_0/C:col10/1733708618865/Put/seqid=0 2024-12-09T01:43:39,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742499_1675 (size=12151) 2024-12-09T01:43:39,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-09T01:43:39,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:39,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:39,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:39,393 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:39,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:39,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:39,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-09T01:43:39,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-09T01:43:39,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:39,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:39,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:39,546 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:39,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:39,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:39,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708679553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:39,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:39,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:39,557 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:39,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-09T01:43:39,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:39,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:39,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:39,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:39,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:39,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-09T01:43:39,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/416b71c4f039476f98cc0493575ac6d5 2024-12-09T01:43:39,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/693c46b1840a4f2893ca4b633a221d7e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/693c46b1840a4f2893ca4b633a221d7e 2024-12-09T01:43:39,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/693c46b1840a4f2893ca4b633a221d7e, entries=200, sequenceid=252, filesize=38.8 K 2024-12-09T01:43:39,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/9d03af4f58d944cc8e062b3e3e414de1 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/9d03af4f58d944cc8e062b3e3e414de1 2024-12-09T01:43:39,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/9d03af4f58d944cc8e062b3e3e414de1, entries=150, sequenceid=252, filesize=11.9 K 2024-12-09T01:43:39,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/416b71c4f039476f98cc0493575ac6d5 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/416b71c4f039476f98cc0493575ac6d5 2024-12-09T01:43:39,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/416b71c4f039476f98cc0493575ac6d5, entries=150, sequenceid=252, filesize=11.9 K 2024-12-09T01:43:39,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 45d234ddf430a4159b8bc5324edd88bc in 882ms, sequenceid=252, compaction requested=true 2024-12-09T01:43:39,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:39,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:43:39,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:39,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:43:39,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:39,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:43:39,749 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:39,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:39,749 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:39,750 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:39,750 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102437 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:39,750 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/A is initiating minor compaction (all files) 2024-12-09T01:43:39,750 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/B is initiating minor compaction (all files) 2024-12-09T01:43:39,750 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/A in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:39,750 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/B in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:39,750 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/06e15ecea86343fbab386b1659f9d69c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2918bb26cd104435acdea2a6a24d1e99, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/693c46b1840a4f2893ca4b633a221d7e] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=100.0 K 2024-12-09T01:43:39,750 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/6663fa7bd87d48e99a28f9c4b8a0e426, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/0c549fa6844144b1a84619c254f9d8af, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/9d03af4f58d944cc8e062b3e3e414de1] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=36.1 K 2024-12-09T01:43:39,750 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:39,750 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. files: [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/06e15ecea86343fbab386b1659f9d69c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2918bb26cd104435acdea2a6a24d1e99, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/693c46b1840a4f2893ca4b633a221d7e] 2024-12-09T01:43:39,750 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 6663fa7bd87d48e99a28f9c4b8a0e426, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733708615060 2024-12-09T01:43:39,751 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06e15ecea86343fbab386b1659f9d69c, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733708615060 2024-12-09T01:43:39,751 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c549fa6844144b1a84619c254f9d8af, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733708616932 2024-12-09T01:43:39,751 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2918bb26cd104435acdea2a6a24d1e99, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733708616932 2024-12-09T01:43:39,751 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 693c46b1840a4f2893ca4b633a221d7e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733708618213 2024-12-09T01:43:39,751 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d03af4f58d944cc8e062b3e3e414de1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733708618213 2024-12-09T01:43:39,758 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:39,758 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#B#compaction#580 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:39,759 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/8e4dfc52ab354a118d7981a3fccdedd9 is 50, key is test_row_0/B:col10/1733708618865/Put/seqid=0 2024-12-09T01:43:39,760 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209f1c89e9fc370440bb9462bb705c13b66_45d234ddf430a4159b8bc5324edd88bc store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:39,762 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209f1c89e9fc370440bb9462bb705c13b66_45d234ddf430a4159b8bc5324edd88bc, store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:39,763 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209f1c89e9fc370440bb9462bb705c13b66_45d234ddf430a4159b8bc5324edd88bc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:39,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742500_1676 (size=12731) 2024-12-09T01:43:39,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742501_1677 (size=4469) 2024-12-09T01:43:39,770 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#A#compaction#581 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:39,771 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/1d8d9b3e6aff4b639e8b6189e4b88703 is 175, key is test_row_0/A:col10/1733708618865/Put/seqid=0 2024-12-09T01:43:39,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742502_1678 (size=31685) 2024-12-09T01:43:39,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-12-09T01:43:39,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:39,852 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 45d234ddf430a4159b8bc5324edd88bc 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-12-09T01:43:39,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=A 2024-12-09T01:43:39,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:39,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=B 2024-12-09T01:43:39,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:39,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=C 2024-12-09T01:43:39,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:39,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209569cd10521104baa9c20bc2fda44e672_45d234ddf430a4159b8bc5324edd88bc is 50, key is test_row_0/A:col10/1733708618889/Put/seqid=0 2024-12-09T01:43:39,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742503_1679 (size=12454) 2024-12-09T01:43:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:40,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:40,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708680151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:40,157 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:40,169 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/8e4dfc52ab354a118d7981a3fccdedd9 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/8e4dfc52ab354a118d7981a3fccdedd9 2024-12-09T01:43:40,173 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/B of 45d234ddf430a4159b8bc5324edd88bc into 8e4dfc52ab354a118d7981a3fccdedd9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:40,173 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:40,173 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/B, priority=13, startTime=1733708619749; duration=0sec 2024-12-09T01:43:40,173 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:40,173 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:B 2024-12-09T01:43:40,173 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:40,174 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:40,174 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/C is initiating minor compaction (all files) 2024-12-09T01:43:40,174 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/C in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:40,174 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/bf9c40a57eba4756bc6136a4dd2c1792, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/f55df748e4f14cc5beaadd668cf55904, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/416b71c4f039476f98cc0493575ac6d5] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=36.1 K 2024-12-09T01:43:40,175 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting bf9c40a57eba4756bc6136a4dd2c1792, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1733708615060 2024-12-09T01:43:40,176 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting f55df748e4f14cc5beaadd668cf55904, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1733708616932 2024-12-09T01:43:40,176 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 416b71c4f039476f98cc0493575ac6d5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733708618213 2024-12-09T01:43:40,179 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/1d8d9b3e6aff4b639e8b6189e4b88703 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/1d8d9b3e6aff4b639e8b6189e4b88703 2024-12-09T01:43:40,183 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/A of 45d234ddf430a4159b8bc5324edd88bc into 1d8d9b3e6aff4b639e8b6189e4b88703(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:40,183 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:40,184 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/A, priority=13, startTime=1733708619749; duration=0sec 2024-12-09T01:43:40,184 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:40,184 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:A 2024-12-09T01:43:40,186 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#C#compaction#583 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:40,187 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/e70cd97452214059948cdf63b8b9f9e8 is 50, key is test_row_0/C:col10/1733708618865/Put/seqid=0 2024-12-09T01:43:40,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742504_1680 (size=12731) 2024-12-09T01:43:40,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-09T01:43:40,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:40,265 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209569cd10521104baa9c20bc2fda44e672_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209569cd10521104baa9c20bc2fda44e672_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:40,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/edbe63e5e053402485e9d1db64973f79, store: [table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:40,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/edbe63e5e053402485e9d1db64973f79 is 175, key is test_row_0/A:col10/1733708618889/Put/seqid=0 2024-12-09T01:43:40,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:40,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708680263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:40,267 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:40,267 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:40,267 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:40,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742505_1681 (size=31255) 2024-12-09T01:43:40,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:40,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708680474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:40,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:40,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:40,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:40,598 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/e70cd97452214059948cdf63b8b9f9e8 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/e70cd97452214059948cdf63b8b9f9e8 2024-12-09T01:43:40,603 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/C of 45d234ddf430a4159b8bc5324edd88bc into e70cd97452214059948cdf63b8b9f9e8(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:40,603 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:40,603 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/C, priority=13, startTime=1733708619749; duration=0sec 2024-12-09T01:43:40,603 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:40,603 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:C 2024-12-09T01:43:40,671 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=266, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/edbe63e5e053402485e9d1db64973f79 2024-12-09T01:43:40,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/c983263f477b4a80b8e69990da7783cb is 50, key is test_row_0/B:col10/1733708618889/Put/seqid=0 2024-12-09T01:43:40,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742506_1682 (size=12301) 2024-12-09T01:43:40,682 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/c983263f477b4a80b8e69990da7783cb 2024-12-09T01:43:40,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/5137b72cb056476f9697558c39453dfc is 50, key is test_row_0/C:col10/1733708618889/Put/seqid=0 2024-12-09T01:43:40,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742507_1683 (size=12301) 2024-12-09T01:43:40,696 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=266 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/5137b72cb056476f9697558c39453dfc 2024-12-09T01:43:40,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/edbe63e5e053402485e9d1db64973f79 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/edbe63e5e053402485e9d1db64973f79 2024-12-09T01:43:40,704 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/edbe63e5e053402485e9d1db64973f79, entries=150, sequenceid=266, filesize=30.5 K 2024-12-09T01:43:40,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/c983263f477b4a80b8e69990da7783cb as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/c983263f477b4a80b8e69990da7783cb 2024-12-09T01:43:40,708 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/c983263f477b4a80b8e69990da7783cb, entries=150, sequenceid=266, filesize=12.0 K 2024-12-09T01:43:40,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/5137b72cb056476f9697558c39453dfc as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/5137b72cb056476f9697558c39453dfc 2024-12-09T01:43:40,713 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/5137b72cb056476f9697558c39453dfc, entries=150, sequenceid=266, filesize=12.0 K 2024-12-09T01:43:40,713 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 45d234ddf430a4159b8bc5324edd88bc in 861ms, sequenceid=266, compaction requested=false 2024-12-09T01:43:40,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:40,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:40,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-12-09T01:43:40,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-12-09T01:43:40,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-12-09T01:43:40,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6280 sec 2024-12-09T01:43:40,724 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 1.6340 sec 2024-12-09T01:43:40,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:40,786 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45d234ddf430a4159b8bc5324edd88bc 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-12-09T01:43:40,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=A 2024-12-09T01:43:40,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:40,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=B 2024-12-09T01:43:40,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:40,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=C 2024-12-09T01:43:40,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:40,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b32fe2339f994a029d7f88527e5f0215_45d234ddf430a4159b8bc5324edd88bc is 50, key is test_row_0/A:col10/1733708620136/Put/seqid=0 2024-12-09T01:43:40,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742508_1684 (size=14994) 2024-12-09T01:43:40,798 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:40,802 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b32fe2339f994a029d7f88527e5f0215_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b32fe2339f994a029d7f88527e5f0215_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:40,803 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/7f18b6210443438e98ecbffa187564de, store: [table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:40,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/7f18b6210443438e98ecbffa187564de is 175, key is test_row_0/A:col10/1733708620136/Put/seqid=0 2024-12-09T01:43:40,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742509_1685 (size=39949) 2024-12-09T01:43:40,808 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/7f18b6210443438e98ecbffa187564de 2024-12-09T01:43:40,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/e4730e3bdd124813badcb123a05da567 is 50, key is test_row_0/B:col10/1733708620136/Put/seqid=0 2024-12-09T01:43:40,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742510_1686 (size=12301) 2024-12-09T01:43:40,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:40,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708680834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:40,839 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:40,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:40,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708680944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:40,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:40,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:40,946 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:41,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:41,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708681153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:41,158 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:41,158 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:41,158 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:41,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/e4730e3bdd124813badcb123a05da567 2024-12-09T01:43:41,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/55600e481ac44e4e901ce7d499ad4d3e is 50, key is test_row_0/C:col10/1733708620136/Put/seqid=0 2024-12-09T01:43:41,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742511_1687 (size=12301) 2024-12-09T01:43:41,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/55600e481ac44e4e901ce7d499ad4d3e 2024-12-09T01:43:41,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/7f18b6210443438e98ecbffa187564de as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/7f18b6210443438e98ecbffa187564de 2024-12-09T01:43:41,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/7f18b6210443438e98ecbffa187564de, entries=200, sequenceid=288, filesize=39.0 K 2024-12-09T01:43:41,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/e4730e3bdd124813badcb123a05da567 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/e4730e3bdd124813badcb123a05da567 2024-12-09T01:43:41,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/e4730e3bdd124813badcb123a05da567, entries=150, sequenceid=288, filesize=12.0 K 2024-12-09T01:43:41,239 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/55600e481ac44e4e901ce7d499ad4d3e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/55600e481ac44e4e901ce7d499ad4d3e 2024-12-09T01:43:41,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/55600e481ac44e4e901ce7d499ad4d3e, entries=150, sequenceid=288, filesize=12.0 K 2024-12-09T01:43:41,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-09T01:43:41,243 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:41,244 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 45d234ddf430a4159b8bc5324edd88bc in 457ms, sequenceid=288, compaction requested=true 2024-12-09T01:43:41,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:41,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:43:41,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:41,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:43:41,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:41,244 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:41,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:43:41,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-12-09T01:43:41,244 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:43:41,245 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:41,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-12-09T01:43:41,245 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:41,246 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/A is initiating minor compaction (all files) 2024-12-09T01:43:41,246 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/A in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:41,246 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/1d8d9b3e6aff4b639e8b6189e4b88703, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/edbe63e5e053402485e9d1db64973f79, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/7f18b6210443438e98ecbffa187564de] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=100.5 K 2024-12-09T01:43:41,246 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:41,246 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. files: [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/1d8d9b3e6aff4b639e8b6189e4b88703, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/edbe63e5e053402485e9d1db64973f79, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/7f18b6210443438e98ecbffa187564de] 2024-12-09T01:43:41,246 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d8d9b3e6aff4b639e8b6189e4b88703, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733708618213 2024-12-09T01:43:41,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T01:43:41,246 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:43:41,247 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting edbe63e5e053402485e9d1db64973f79, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733708618889 2024-12-09T01:43:41,247 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f18b6210443438e98ecbffa187564de, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733708620136 2024-12-09T01:43:41,247 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:41,247 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/B is initiating minor compaction (all files) 2024-12-09T01:43:41,247 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/B in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:41,247 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/8e4dfc52ab354a118d7981a3fccdedd9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/c983263f477b4a80b8e69990da7783cb, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/e4730e3bdd124813badcb123a05da567] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=36.5 K 2024-12-09T01:43:41,247 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:43:41,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:43:41,247 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e4dfc52ab354a118d7981a3fccdedd9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733708618213 2024-12-09T01:43:41,248 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting c983263f477b4a80b8e69990da7783cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733708618889 2024-12-09T01:43:41,254 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:41,256 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting e4730e3bdd124813badcb123a05da567, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733708620136 2024-12-09T01:43:41,256 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024120995b186c74ce4400c9b63e19be630294d_45d234ddf430a4159b8bc5324edd88bc store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:41,258 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024120995b186c74ce4400c9b63e19be630294d_45d234ddf430a4159b8bc5324edd88bc, store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:41,258 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120995b186c74ce4400c9b63e19be630294d_45d234ddf430a4159b8bc5324edd88bc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:41,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742512_1688 (size=4469) 2024-12-09T01:43:41,287 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#B#compaction#590 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:41,287 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#A#compaction#589 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:41,288 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/409600b38c3e49c18aa4da7dfdf201d5 is 50, key is test_row_0/B:col10/1733708620136/Put/seqid=0 2024-12-09T01:43:41,288 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/2eda9906e3404aa6a3c3c20769eb0aff is 175, key is test_row_0/A:col10/1733708620136/Put/seqid=0 2024-12-09T01:43:41,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742514_1690 (size=31937) 2024-12-09T01:43:41,299 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/2eda9906e3404aa6a3c3c20769eb0aff as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2eda9906e3404aa6a3c3c20769eb0aff 2024-12-09T01:43:41,305 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/A of 45d234ddf430a4159b8bc5324edd88bc into 2eda9906e3404aa6a3c3c20769eb0aff(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:41,305 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:41,305 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/A, priority=13, startTime=1733708621244; duration=0sec 2024-12-09T01:43:41,305 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:41,305 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:A 2024-12-09T01:43:41,305 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:41,307 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:41,307 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/C is initiating minor compaction (all files) 2024-12-09T01:43:41,307 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/C in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:41,308 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/e70cd97452214059948cdf63b8b9f9e8, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/5137b72cb056476f9697558c39453dfc, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/55600e481ac44e4e901ce7d499ad4d3e] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=36.5 K 2024-12-09T01:43:41,308 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting e70cd97452214059948cdf63b8b9f9e8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1733708618213 2024-12-09T01:43:41,308 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5137b72cb056476f9697558c39453dfc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=266, earliestPutTs=1733708618889 2024-12-09T01:43:41,309 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55600e481ac44e4e901ce7d499ad4d3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733708620136 2024-12-09T01:43:41,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742513_1689 (size=12983) 2024-12-09T01:43:41,317 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#C#compaction#591 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:41,317 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/261de500be644ba191e1b81d2bc45b9e is 50, key is test_row_0/C:col10/1733708620136/Put/seqid=0 2024-12-09T01:43:41,321 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/409600b38c3e49c18aa4da7dfdf201d5 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/409600b38c3e49c18aa4da7dfdf201d5 2024-12-09T01:43:41,326 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/B of 45d234ddf430a4159b8bc5324edd88bc into 409600b38c3e49c18aa4da7dfdf201d5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:41,326 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:41,326 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/B, priority=13, startTime=1733708621244; duration=0sec 2024-12-09T01:43:41,327 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:41,327 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:B 2024-12-09T01:43:41,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742515_1691 (size=12983) 2024-12-09T01:43:41,339 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/261de500be644ba191e1b81d2bc45b9e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/261de500be644ba191e1b81d2bc45b9e 2024-12-09T01:43:41,343 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/C of 45d234ddf430a4159b8bc5324edd88bc into 261de500be644ba191e1b81d2bc45b9e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:41,343 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:41,343 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/C, priority=13, startTime=1733708621244; duration=0sec 2024-12-09T01:43:41,343 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:41,343 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:C 2024-12-09T01:43:41,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T01:43:41,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-12-09T01:43:41,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:41,401 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 45d234ddf430a4159b8bc5324edd88bc 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-12-09T01:43:41,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=A 2024-12-09T01:43:41,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:41,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=B 2024-12-09T01:43:41,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:41,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=C 2024-12-09T01:43:41,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:41,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209dcf372c8ed854ccba3f3d1217e639374_45d234ddf430a4159b8bc5324edd88bc is 50, key is test_row_0/A:col10/1733708620818/Put/seqid=0 2024-12-09T01:43:41,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742516_1692 (size=12454) 2024-12-09T01:43:41,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:41,433 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209dcf372c8ed854ccba3f3d1217e639374_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209dcf372c8ed854ccba3f3d1217e639374_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:41,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/6a03e6d6167a4a21a8b4e34d74580ee5, store: [table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:41,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/6a03e6d6167a4a21a8b4e34d74580ee5 is 175, key is test_row_0/A:col10/1733708620818/Put/seqid=0 2024-12-09T01:43:41,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742517_1693 (size=31255) 2024-12-09T01:43:41,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:41,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:41,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:41,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708681556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:41,560 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:41,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:41,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:41,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T01:43:41,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:41,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708681663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:41,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:41,839 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/6a03e6d6167a4a21a8b4e34d74580ee5 2024-12-09T01:43:41,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/9dffe96fd8c44d0b95a81c201b89f34f is 50, key is test_row_0/B:col10/1733708620818/Put/seqid=0 2024-12-09T01:43:41,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742518_1694 (size=12301) 2024-12-09T01:43:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T01:43:41,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708681874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:42,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708682183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:42,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:42,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:42,186 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:42,249 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/9dffe96fd8c44d0b95a81c201b89f34f 2024-12-09T01:43:42,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/65f9c09d665a4952a0d5ecf4ec937a14 is 50, key is test_row_0/C:col10/1733708620818/Put/seqid=0 2024-12-09T01:43:42,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742519_1695 (size=12301) 2024-12-09T01:43:42,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T01:43:42,659 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/65f9c09d665a4952a0d5ecf4ec937a14 2024-12-09T01:43:42,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/6a03e6d6167a4a21a8b4e34d74580ee5 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/6a03e6d6167a4a21a8b4e34d74580ee5 2024-12-09T01:43:42,666 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/6a03e6d6167a4a21a8b4e34d74580ee5, entries=150, sequenceid=304, filesize=30.5 K 2024-12-09T01:43:42,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/9dffe96fd8c44d0b95a81c201b89f34f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/9dffe96fd8c44d0b95a81c201b89f34f 2024-12-09T01:43:42,670 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/9dffe96fd8c44d0b95a81c201b89f34f, entries=150, sequenceid=304, filesize=12.0 K 2024-12-09T01:43:42,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/65f9c09d665a4952a0d5ecf4ec937a14 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/65f9c09d665a4952a0d5ecf4ec937a14 2024-12-09T01:43:42,674 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/65f9c09d665a4952a0d5ecf4ec937a14, entries=150, sequenceid=304, filesize=12.0 K 2024-12-09T01:43:42,675 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 45d234ddf430a4159b8bc5324edd88bc in 1274ms, sequenceid=304, compaction requested=false 2024-12-09T01:43:42,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:42,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:42,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-12-09T01:43:42,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-12-09T01:43:42,679 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-12-09T01:43:42,679 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4300 sec 2024-12-09T01:43:42,681 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 1.4360 sec 2024-12-09T01:43:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:42,698 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45d234ddf430a4159b8bc5324edd88bc 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-12-09T01:43:42,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=A 2024-12-09T01:43:42,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:42,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=B 2024-12-09T01:43:42,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:42,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=C 2024-12-09T01:43:42,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:42,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120935a310fd5b2e43bbbfafd20ce557be90_45d234ddf430a4159b8bc5324edd88bc is 50, key is test_row_0/A:col10/1733708622696/Put/seqid=0 2024-12-09T01:43:42,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742520_1696 (size=14994) 2024-12-09T01:43:42,710 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:42,713 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024120935a310fd5b2e43bbbfafd20ce557be90_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120935a310fd5b2e43bbbfafd20ce557be90_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:42,714 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/0951c63fd0744ab8939ecd6480c86c3b, store: [table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:42,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/0951c63fd0744ab8939ecd6480c86c3b is 175, key is test_row_0/A:col10/1733708622696/Put/seqid=0 2024-12-09T01:43:42,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742521_1697 (size=39949) 2024-12-09T01:43:42,741 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:42,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708682737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:42,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:42,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:42,742 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:42,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:42,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708682843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:42,845 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:42,845 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:42,845 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:43,056 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:43,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708683053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:43,057 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:43,120 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/0951c63fd0744ab8939ecd6480c86c3b 2024-12-09T01:43:43,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/e92e4175b31f4686abf6559fd0bf470b is 50, key is test_row_0/B:col10/1733708622696/Put/seqid=0 2024-12-09T01:43:43,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742522_1698 (size=12301) 2024-12-09T01:43:43,133 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/e92e4175b31f4686abf6559fd0bf470b 2024-12-09T01:43:43,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/26b3cc34adeb48ba937e7ecedfdd4954 is 50, key is test_row_0/C:col10/1733708622696/Put/seqid=0 2024-12-09T01:43:43,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742523_1699 (size=12301) 2024-12-09T01:43:43,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40020 deadline: 1733708683313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:43,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39966 deadline: 1733708683313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:43,317 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_1' in 45d234ddf430a4159b8bc5324edd88bc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:43:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:43,317 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_1' in 45d234ddf430a4159b8bc5324edd88bc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:43:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:43,318 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:43,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:43,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39980 deadline: 1733708683334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:43,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:43,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:39992 deadline: 1733708683334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:43,337 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_2' in 45d234ddf430a4159b8bc5324edd88bc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:43:43,337 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:43,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:43,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:43,337 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to ef6f18c58dc9,38627,1733708453943 for 'test_row_2' in 45d234ddf430a4159b8bc5324edd88bc of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-12-09T01:43:43,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:43,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:43,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:43,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:43,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708683363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:43,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:43,366 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:43,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-12-09T01:43:43,394 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:43,394 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-12-09T01:43:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-12-09T01:43:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-09T01:43:43,396 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-12-09T01:43:43,397 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-12-09T01:43:43,397 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-12-09T01:43:43,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-09T01:43:43,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/26b3cc34adeb48ba937e7ecedfdd4954 2024-12-09T01:43:43,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-09T01:43:43,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:43,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:43,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:43,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:43,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:43,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T01:43:43,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/0951c63fd0744ab8939ecd6480c86c3b as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/0951c63fd0744ab8939ecd6480c86c3b 2024-12-09T01:43:43,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/0951c63fd0744ab8939ecd6480c86c3b, entries=200, sequenceid=324, filesize=39.0 K 2024-12-09T01:43:43,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/e92e4175b31f4686abf6559fd0bf470b as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/e92e4175b31f4686abf6559fd0bf470b 2024-12-09T01:43:43,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/e92e4175b31f4686abf6559fd0bf470b, entries=150, sequenceid=324, filesize=12.0 K 2024-12-09T01:43:43,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/26b3cc34adeb48ba937e7ecedfdd4954 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/26b3cc34adeb48ba937e7ecedfdd4954 2024-12-09T01:43:43,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/26b3cc34adeb48ba937e7ecedfdd4954, entries=150, sequenceid=324, filesize=12.0 K 2024-12-09T01:43:43,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 45d234ddf430a4159b8bc5324edd88bc in 866ms, sequenceid=324, compaction requested=true 2024-12-09T01:43:43,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:43,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:43:43,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:43,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:43:43,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:43,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:43:43,563 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:43,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:43,563 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:43,564 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:43,564 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:43,564 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/B is initiating minor compaction (all files) 2024-12-09T01:43:43,564 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/A is initiating minor compaction (all files) 2024-12-09T01:43:43,564 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/B in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:43,564 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/A in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:43,564 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/409600b38c3e49c18aa4da7dfdf201d5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/9dffe96fd8c44d0b95a81c201b89f34f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/e92e4175b31f4686abf6559fd0bf470b] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=36.7 K 2024-12-09T01:43:43,564 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2eda9906e3404aa6a3c3c20769eb0aff, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/6a03e6d6167a4a21a8b4e34d74580ee5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/0951c63fd0744ab8939ecd6480c86c3b] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=100.7 K 2024-12-09T01:43:43,564 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:43,564 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. files: [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2eda9906e3404aa6a3c3c20769eb0aff, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/6a03e6d6167a4a21a8b4e34d74580ee5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/0951c63fd0744ab8939ecd6480c86c3b] 2024-12-09T01:43:43,565 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 409600b38c3e49c18aa4da7dfdf201d5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733708620136 2024-12-09T01:43:43,565 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2eda9906e3404aa6a3c3c20769eb0aff, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733708620136 2024-12-09T01:43:43,565 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dffe96fd8c44d0b95a81c201b89f34f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733708620813 2024-12-09T01:43:43,565 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a03e6d6167a4a21a8b4e34d74580ee5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733708620813 2024-12-09T01:43:43,565 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0951c63fd0744ab8939ecd6480c86c3b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733708621541 2024-12-09T01:43:43,566 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting e92e4175b31f4686abf6559fd0bf470b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733708621547 2024-12-09T01:43:43,571 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:43,573 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241209c163a33e138d4e7abc074274a89afc74_45d234ddf430a4159b8bc5324edd88bc store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:43,574 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#B#compaction#599 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:43,574 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/ba32b5fdb5fe4b82a5761070cb25b7cb is 50, key is test_row_0/B:col10/1733708622696/Put/seqid=0 2024-12-09T01:43:43,575 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241209c163a33e138d4e7abc074274a89afc74_45d234ddf430a4159b8bc5324edd88bc, store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:43,575 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209c163a33e138d4e7abc074274a89afc74_45d234ddf430a4159b8bc5324edd88bc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:43,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742524_1700 (size=13085) 2024-12-09T01:43:43,588 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/ba32b5fdb5fe4b82a5761070cb25b7cb as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/ba32b5fdb5fe4b82a5761070cb25b7cb 2024-12-09T01:43:43,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742525_1701 (size=4469) 2024-12-09T01:43:43,592 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/B of 45d234ddf430a4159b8bc5324edd88bc into ba32b5fdb5fe4b82a5761070cb25b7cb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:43,592 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:43,592 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/B, priority=13, startTime=1733708623563; duration=0sec 2024-12-09T01:43:43,593 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:43,593 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:B 2024-12-09T01:43:43,593 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:43,593 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:43,593 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/C is initiating minor compaction (all files) 2024-12-09T01:43:43,594 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/C in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:43,594 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/261de500be644ba191e1b81d2bc45b9e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/65f9c09d665a4952a0d5ecf4ec937a14, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/26b3cc34adeb48ba937e7ecedfdd4954] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=36.7 K 2024-12-09T01:43:43,594 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 261de500be644ba191e1b81d2bc45b9e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1733708620136 2024-12-09T01:43:43,595 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 65f9c09d665a4952a0d5ecf4ec937a14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1733708620813 2024-12-09T01:43:43,595 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 26b3cc34adeb48ba937e7ecedfdd4954, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733708621547 2024-12-09T01:43:43,603 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#C#compaction#600 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:43,603 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/cdd156ed72234d31866bbb58d1ad13b6 is 50, key is test_row_0/C:col10/1733708622696/Put/seqid=0 2024-12-09T01:43:43,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742526_1702 (size=13085) 2024-12-09T01:43:43,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38627 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-12-09T01:43:43,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:43,703 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 45d234ddf430a4159b8bc5324edd88bc 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-12-09T01:43:43,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=A 2024-12-09T01:43:43,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:43,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=B 2024-12-09T01:43:43,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:43,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=C 2024-12-09T01:43:43,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:43,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098d93ce47e1a5459285869db68c863ec8_45d234ddf430a4159b8bc5324edd88bc is 50, key is test_row_0/A:col10/1733708622729/Put/seqid=0 2024-12-09T01:43:43,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-09T01:43:43,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742527_1703 (size=12454) 2024-12-09T01:43:43,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:43,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. as already flushing 2024-12-09T01:43:43,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708683960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:43,965 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:43,965 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:43,965 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:43,992 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#A#compaction#598 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:43,993 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/1a41b1b2842244228f1e8bb68de5e5ba is 175, key is test_row_0/A:col10/1733708622696/Put/seqid=0 2024-12-09T01:43:44,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742528_1704 (size=32039) 2024-12-09T01:43:44,019 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/cdd156ed72234d31866bbb58d1ad13b6 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/cdd156ed72234d31866bbb58d1ad13b6 2024-12-09T01:43:44,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-09T01:43:44,025 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/1a41b1b2842244228f1e8bb68de5e5ba as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/1a41b1b2842244228f1e8bb68de5e5ba 2024-12-09T01:43:44,026 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/C of 45d234ddf430a4159b8bc5324edd88bc into cdd156ed72234d31866bbb58d1ad13b6(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:44,026 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:44,026 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/C, priority=13, startTime=1733708623563; duration=0sec 2024-12-09T01:43:44,026 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:44,026 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:C 2024-12-09T01:43:44,030 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/A of 45d234ddf430a4159b8bc5324edd88bc into 1a41b1b2842244228f1e8bb68de5e5ba(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:44,030 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:44,030 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/A, priority=13, startTime=1733708623563; duration=0sec 2024-12-09T01:43:44,030 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:44,030 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:A 2024-12-09T01:43:44,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:44,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708684073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:44,077 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:44,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:44,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:44,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:44,121 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412098d93ce47e1a5459285869db68c863ec8_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098d93ce47e1a5459285869db68c863ec8_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:44,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/d3fb6e01a51748f5b9b864364f67804f, store: [table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:44,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/d3fb6e01a51748f5b9b864364f67804f is 175, key is test_row_0/A:col10/1733708622729/Put/seqid=0 2024-12-09T01:43:44,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742529_1705 (size=31255) 2024-12-09T01:43:44,288 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708684283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:44,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-09T01:43:44,559 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=337, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/d3fb6e01a51748f5b9b864364f67804f 2024-12-09T01:43:44,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/d80696dcdf0f49deb2fc1d633a2fc74c is 50, key is test_row_0/B:col10/1733708622729/Put/seqid=0 2024-12-09T01:43:44,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-12-09T01:43:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:40044 deadline: 1733708684594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:44,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 , the old value is region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:44,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-09T01:43:44,597 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., hostname=ef6f18c58dc9,38627,1733708453943, seqNum=5 because the exception is null or not the one we care about 2024-12-09T01:43:44,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742530_1706 (size=12301) 2024-12-09T01:43:45,010 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/d80696dcdf0f49deb2fc1d633a2fc74c 2024-12-09T01:43:45,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/797d779306b246648892e547cf9549f2 is 50, key is test_row_0/C:col10/1733708622729/Put/seqid=0 2024-12-09T01:43:45,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742531_1707 (size=12301) 2024-12-09T01:43:45,030 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=337 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/797d779306b246648892e547cf9549f2 2024-12-09T01:43:45,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/d3fb6e01a51748f5b9b864364f67804f as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/d3fb6e01a51748f5b9b864364f67804f 2024-12-09T01:43:45,038 INFO [Thread-2689 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2689. 2024-12-09T01:43:45,038 DEBUG [Thread-2689 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:43:45,038 DEBUG [Thread-2689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:45,038 DEBUG [Thread-2689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:45,038 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/d3fb6e01a51748f5b9b864364f67804f, entries=150, sequenceid=337, filesize=30.5 K 2024-12-09T01:43:45,038 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:43:45,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/d80696dcdf0f49deb2fc1d633a2fc74c as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/d80696dcdf0f49deb2fc1d633a2fc74c 2024-12-09T01:43:45,040 INFO [Thread-2693 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2693. 2024-12-09T01:43:45,040 DEBUG [Thread-2693 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:43:45,040 DEBUG [Thread-2693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:45,040 DEBUG [Thread-2693 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:45,040 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:43:45,040 INFO [Thread-2685 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2685. 2024-12-09T01:43:45,040 DEBUG [Thread-2685 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:43:45,040 DEBUG [Thread-2685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:45,041 DEBUG [Thread-2685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:45,041 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:43:45,043 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/d80696dcdf0f49deb2fc1d633a2fc74c, entries=150, sequenceid=337, filesize=12.0 K 2024-12-09T01:43:45,044 INFO [Thread-2687 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2687. 2024-12-09T01:43:45,044 DEBUG [Thread-2687 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:43:45,044 DEBUG [Thread-2687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:45,044 DEBUG [Thread-2687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:45,044 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:43:45,044 INFO [Thread-2691 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2691. 2024-12-09T01:43:45,044 DEBUG [Thread-2691 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:43:45,044 DEBUG [Thread-2691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:45,045 DEBUG [Thread-2691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:45,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:43:45,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/797d779306b246648892e547cf9549f2 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/797d779306b246648892e547cf9549f2 2024-12-09T01:43:45,053 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/797d779306b246648892e547cf9549f2, entries=150, sequenceid=337, filesize=12.0 K 2024-12-09T01:43:45,054 INFO [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 45d234ddf430a4159b8bc5324edd88bc in 1351ms, sequenceid=337, compaction requested=false 2024-12-09T01:43:45,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:45,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:45,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ef6f18c58dc9:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-12-09T01:43:45,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-12-09T01:43:45,057 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-12-09T01:43:45,057 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6590 sec 2024-12-09T01:43:45,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 1.6640 sec 2024-12-09T01:43:45,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38627 {}] regionserver.HRegion(8855): Flush requested on 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:45,105 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 45d234ddf430a4159b8bc5324edd88bc 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-12-09T01:43:45,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=A 2024-12-09T01:43:45,106 INFO [Thread-2678 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2678. 2024-12-09T01:43:45,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:45,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=B 2024-12-09T01:43:45,106 DEBUG [Thread-2678 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:43:45,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:45,106 DEBUG [Thread-2678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:45,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=C 2024-12-09T01:43:45,106 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-12-09T01:43:45,106 DEBUG [Thread-2678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:45,106 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:43:45,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209fd16d76628c34768860a7640b4043338_45d234ddf430a4159b8bc5324edd88bc is 50, key is test_row_0/A:col10/1733708623959/Put/seqid=0 2024-12-09T01:43:45,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742532_1708 (size=12454) 2024-12-09T01:43:45,518 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:45,521 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209fd16d76628c34768860a7640b4043338_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209fd16d76628c34768860a7640b4043338_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:45,521 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/2fb4bbef2a80435cb31c7e245bc27315, store: [table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:45,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/2fb4bbef2a80435cb31c7e245bc27315 is 175, key is test_row_0/A:col10/1733708623959/Put/seqid=0 2024-12-09T01:43:45,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742533_1709 (size=31255) 2024-12-09T01:43:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-12-09T01:43:45,544 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:45,926 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/2fb4bbef2a80435cb31c7e245bc27315 2024-12-09T01:43:45,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/8867c5db3cec4e68918bcc9b111d0d54 is 50, key is test_row_0/B:col10/1733708623959/Put/seqid=0 2024-12-09T01:43:45,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742534_1710 (size=12301) 2024-12-09T01:43:46,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/8867c5db3cec4e68918bcc9b111d0d54 2024-12-09T01:43:46,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/08efa270af60458791a9a34b49d45cb6 is 50, key is test_row_0/C:col10/1733708623959/Put/seqid=0 2024-12-09T01:43:46,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742535_1711 (size=12301) 2024-12-09T01:43:46,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/08efa270af60458791a9a34b49d45cb6 2024-12-09T01:43:46,748 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/2fb4bbef2a80435cb31c7e245bc27315 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2fb4bbef2a80435cb31c7e245bc27315 2024-12-09T01:43:46,751 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2fb4bbef2a80435cb31c7e245bc27315, entries=150, sequenceid=360, filesize=30.5 K 2024-12-09T01:43:46,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/8867c5db3cec4e68918bcc9b111d0d54 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/8867c5db3cec4e68918bcc9b111d0d54 2024-12-09T01:43:46,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/8867c5db3cec4e68918bcc9b111d0d54, entries=150, sequenceid=360, filesize=12.0 K 2024-12-09T01:43:46,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/08efa270af60458791a9a34b49d45cb6 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/08efa270af60458791a9a34b49d45cb6 2024-12-09T01:43:46,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/08efa270af60458791a9a34b49d45cb6, entries=150, sequenceid=360, filesize=12.0 K 2024-12-09T01:43:46,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=0 B/0 for 45d234ddf430a4159b8bc5324edd88bc in 1654ms, sequenceid=360, compaction requested=true 2024-12-09T01:43:46,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:46,760 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:46,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:A, priority=-2147483648, current under compaction store size is 1 2024-12-09T01:43:46,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:46,760 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:46,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:B, priority=-2147483648, current under compaction store size is 2 2024-12-09T01:43:46,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:46,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 45d234ddf430a4159b8bc5324edd88bc:C, priority=-2147483648, current under compaction store size is 3 2024-12-09T01:43:46,760 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:46,761 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:46,761 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/A is initiating minor compaction (all files) 2024-12-09T01:43:46,761 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/A in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:46,761 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/1a41b1b2842244228f1e8bb68de5e5ba, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/d3fb6e01a51748f5b9b864364f67804f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2fb4bbef2a80435cb31c7e245bc27315] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=92.3 K 2024-12-09T01:43:46,761 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:46,761 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. files: [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/1a41b1b2842244228f1e8bb68de5e5ba, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/d3fb6e01a51748f5b9b864364f67804f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2fb4bbef2a80435cb31c7e245bc27315] 2024-12-09T01:43:46,762 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a41b1b2842244228f1e8bb68de5e5ba, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733708621547 2024-12-09T01:43:46,762 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:46,762 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3fb6e01a51748f5b9b864364f67804f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733708622716 2024-12-09T01:43:46,762 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/B is initiating minor compaction (all files) 2024-12-09T01:43:46,762 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/B in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:46,762 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/ba32b5fdb5fe4b82a5761070cb25b7cb, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/d80696dcdf0f49deb2fc1d633a2fc74c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/8867c5db3cec4e68918bcc9b111d0d54] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=36.8 K 2024-12-09T01:43:46,762 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fb4bbef2a80435cb31c7e245bc27315, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733708623943 2024-12-09T01:43:46,763 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting ba32b5fdb5fe4b82a5761070cb25b7cb, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733708621547 2024-12-09T01:43:46,763 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting d80696dcdf0f49deb2fc1d633a2fc74c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733708622716 2024-12-09T01:43:46,764 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 8867c5db3cec4e68918bcc9b111d0d54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733708623943 2024-12-09T01:43:46,776 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:46,781 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#B#compaction#608 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:46,782 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/c6e71284f7e94e97914b0b477c684c35 is 50, key is test_row_0/B:col10/1733708623959/Put/seqid=0 2024-12-09T01:43:46,787 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202412099f3769e62b0542289d98ffcef70919a4_45d234ddf430a4159b8bc5324edd88bc store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:46,791 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202412099f3769e62b0542289d98ffcef70919a4_45d234ddf430a4159b8bc5324edd88bc, store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:46,791 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202412099f3769e62b0542289d98ffcef70919a4_45d234ddf430a4159b8bc5324edd88bc because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:46,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742536_1712 (size=13187) 2024-12-09T01:43:46,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742537_1713 (size=4469) 2024-12-09T01:43:47,199 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/c6e71284f7e94e97914b0b477c684c35 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/c6e71284f7e94e97914b0b477c684c35 2024-12-09T01:43:47,202 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/B of 45d234ddf430a4159b8bc5324edd88bc into c6e71284f7e94e97914b0b477c684c35(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:47,202 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:47,202 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/B, priority=13, startTime=1733708626760; duration=0sec 2024-12-09T01:43:47,203 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-12-09T01:43:47,203 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:B 2024-12-09T01:43:47,203 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-12-09T01:43:47,203 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-12-09T01:43:47,203 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1541): 45d234ddf430a4159b8bc5324edd88bc/C is initiating minor compaction (all files) 2024-12-09T01:43:47,203 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 45d234ddf430a4159b8bc5324edd88bc/C in TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:47,203 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/cdd156ed72234d31866bbb58d1ad13b6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/797d779306b246648892e547cf9549f2, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/08efa270af60458791a9a34b49d45cb6] into tmpdir=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp, totalSize=36.8 K 2024-12-09T01:43:47,204 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting cdd156ed72234d31866bbb58d1ad13b6, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1733708621547 2024-12-09T01:43:47,204 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 797d779306b246648892e547cf9549f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=337, earliestPutTs=1733708622716 2024-12-09T01:43:47,204 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] compactions.Compactor(225): Compacting 08efa270af60458791a9a34b49d45cb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1733708623943 2024-12-09T01:43:47,206 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#A#compaction#607 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:47,206 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/1b61be849401472d9a4e90ce41d3dc81 is 175, key is test_row_0/A:col10/1733708623959/Put/seqid=0 2024-12-09T01:43:47,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742538_1714 (size=32141) 2024-12-09T01:43:47,209 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 45d234ddf430a4159b8bc5324edd88bc#C#compaction#609 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-09T01:43:47,210 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/eb6aba9b79cc4108b38475821e6c541e is 50, key is test_row_0/C:col10/1733708623959/Put/seqid=0 2024-12-09T01:43:47,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742539_1715 (size=13187) 2024-12-09T01:43:47,217 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/eb6aba9b79cc4108b38475821e6c541e as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/eb6aba9b79cc4108b38475821e6c541e 2024-12-09T01:43:47,220 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/C of 45d234ddf430a4159b8bc5324edd88bc into eb6aba9b79cc4108b38475821e6c541e(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:47,221 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:47,221 INFO [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/C, priority=13, startTime=1733708626760; duration=0sec 2024-12-09T01:43:47,221 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:47,221 DEBUG [RS:0;ef6f18c58dc9:38627-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:C 2024-12-09T01:43:47,613 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/1b61be849401472d9a4e90ce41d3dc81 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/1b61be849401472d9a4e90ce41d3dc81 2024-12-09T01:43:47,616 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 45d234ddf430a4159b8bc5324edd88bc/A of 45d234ddf430a4159b8bc5324edd88bc into 1b61be849401472d9a4e90ce41d3dc81(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-09T01:43:47,616 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 45d234ddf430a4159b8bc5324edd88bc: 2024-12-09T01:43:47,617 INFO [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., storeName=45d234ddf430a4159b8bc5324edd88bc/A, priority=13, startTime=1733708626760; duration=0sec 2024-12-09T01:43:47,617 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-12-09T01:43:47,617 DEBUG [RS:0;ef6f18c58dc9:38627-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 45d234ddf430a4159b8bc5324edd88bc:A 2024-12-09T01:43:51,960 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T01:43:53,345 INFO [Thread-2676 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2676. 2024-12-09T01:43:53,345 DEBUG [Thread-2676 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:43:53,345 DEBUG [Thread-2676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:53,345 DEBUG [Thread-2676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:53,345 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:43:53,355 INFO [Thread-2682 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2682. 2024-12-09T01:43:53,355 DEBUG [Thread-2682 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:43:53,355 DEBUG [Thread-2682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:53,355 DEBUG [Thread-2682 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:53,355 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:43:53,375 INFO [Thread-2674 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2674. 2024-12-09T01:43:53,375 DEBUG [Thread-2674 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:43:53,375 DEBUG [Thread-2674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:53,375 DEBUG [Thread-2674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:53,375 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:43:53,414 INFO [Thread-2680 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2680. 2024-12-09T01:43:53,414 DEBUG [Thread-2680 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-12-09T01:43:53,415 DEBUG [Thread-2680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:53,415 DEBUG [Thread-2680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:53,415 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:43:53,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-12-09T01:43:53,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-12-09T01:43:53,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-12-09T01:43:53,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 126 2024-12-09T01:43:53,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-12-09T01:43:53,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-12-09T01:43:53,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-12-09T01:43:53,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2715 2024-12-09T01:43:53,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2701 2024-12-09T01:43:53,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2633 2024-12-09T01:43:53,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2715 2024-12-09T01:43:53,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2707 2024-12-09T01:43:53,415 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-12-09T01:43:53,415 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T01:43:53,415 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T01:43:53,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:53,415 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:53,415 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:43:53,416 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-12-09T01:43:53,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-12-09T01:43:53,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-09T01:43:53,419 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708633419"}]},"ts":"1733708633419"} 2024-12-09T01:43:53,420 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-12-09T01:43:53,420 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-12-09T01:43:53,421 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-12-09T01:43:53,422 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45d234ddf430a4159b8bc5324edd88bc, UNASSIGN}] 2024-12-09T01:43:53,423 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45d234ddf430a4159b8bc5324edd88bc, UNASSIGN 2024-12-09T01:43:53,423 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=45d234ddf430a4159b8bc5324edd88bc, regionState=CLOSING, regionLocation=ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:53,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=45d234ddf430a4159b8bc5324edd88bc, UNASSIGN because future has completed 2024-12-09T01:43:53,425 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-09T01:43:53,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure 45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943}] 2024-12-09T01:43:53,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-09T01:43:53,577 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(122): Close 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:53,577 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-09T01:43:53,577 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1722): Closing 45d234ddf430a4159b8bc5324edd88bc, disabling compactions & flushes 2024-12-09T01:43:53,577 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:53,577 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:53,577 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. after waiting 0 ms 2024-12-09T01:43:53,577 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:53,577 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(2902): Flushing 45d234ddf430a4159b8bc5324edd88bc 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-12-09T01:43:53,577 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=A 2024-12-09T01:43:53,577 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:53,577 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=B 2024-12-09T01:43:53,578 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:53,578 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 45d234ddf430a4159b8bc5324edd88bc, store=C 2024-12-09T01:43:53,578 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-12-09T01:43:53,582 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b3aaa728ba1e4f77ad125cc0ba6e8f83_45d234ddf430a4159b8bc5324edd88bc is 50, key is test_row_1/A:col10/1733708633413/Put/seqid=0 2024-12-09T01:43:53,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742540_1716 (size=9914) 2024-12-09T01:43:53,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-09T01:43:53,985 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T01:43:53,988 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241209b3aaa728ba1e4f77ad125cc0ba6e8f83_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b3aaa728ba1e4f77ad125cc0ba6e8f83_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:53,989 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/665e671c9ef04d319e2fcd2e2b410e95, store: [table=TestAcidGuarantees family=A region=45d234ddf430a4159b8bc5324edd88bc] 2024-12-09T01:43:53,990 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/665e671c9ef04d319e2fcd2e2b410e95 is 175, key is test_row_1/A:col10/1733708633413/Put/seqid=0 2024-12-09T01:43:53,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742541_1717 (size=22561) 2024-12-09T01:43:54,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-09T01:43:54,393 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=370, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/665e671c9ef04d319e2fcd2e2b410e95 2024-12-09T01:43:54,398 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/bbe9921e9e504b8392371c0c2f21ddd0 is 50, key is test_row_1/B:col10/1733708633413/Put/seqid=0 2024-12-09T01:43:54,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742542_1718 (size=9857) 2024-12-09T01:43:54,402 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/bbe9921e9e504b8392371c0c2f21ddd0 2024-12-09T01:43:54,407 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/d3784f921dd2461289fa4cba6136b838 is 50, key is test_row_1/C:col10/1733708633413/Put/seqid=0 2024-12-09T01:43:54,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742543_1719 (size=9857) 2024-12-09T01:43:54,410 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=370 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/d3784f921dd2461289fa4cba6136b838 2024-12-09T01:43:54,413 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/A/665e671c9ef04d319e2fcd2e2b410e95 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/665e671c9ef04d319e2fcd2e2b410e95 2024-12-09T01:43:54,416 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/665e671c9ef04d319e2fcd2e2b410e95, entries=100, sequenceid=370, filesize=22.0 K 2024-12-09T01:43:54,416 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/B/bbe9921e9e504b8392371c0c2f21ddd0 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/bbe9921e9e504b8392371c0c2f21ddd0 2024-12-09T01:43:54,419 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/bbe9921e9e504b8392371c0c2f21ddd0, entries=100, sequenceid=370, filesize=9.6 K 2024-12-09T01:43:54,420 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/.tmp/C/d3784f921dd2461289fa4cba6136b838 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/d3784f921dd2461289fa4cba6136b838 2024-12-09T01:43:54,422 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/d3784f921dd2461289fa4cba6136b838, entries=100, sequenceid=370, filesize=9.6 K 2024-12-09T01:43:54,423 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 45d234ddf430a4159b8bc5324edd88bc in 846ms, sequenceid=370, compaction requested=false 2024-12-09T01:43:54,424 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/ca9f66c95fec4d87a8129559a638fbc7, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/93bfc537b6ed4c019708848a726eb884, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/f9024eebdc214a9fb7a2c2eb4bffcc68, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/1f1568359e0f4f83a8c41b1d0997eb99, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/8d90d7a849e144c8a0bfb26ffa9d796b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/92dd1c62bed643cf80d0137dd556d0e9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2fb6b5e2e77c46859ae65103addd618e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/8d7e461a404d4cecbd7dafaaf8064c80, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/9ba75082de6f46a49010b565a00108f7, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/c7dbcac6f2c24434a185b202767700b9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/0dc51fe951b04e8fbcabe61133271f35, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/56a881e3565843bc8b85b94335b10b80, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/e32d8cccd1854c5597a40acf0a6f7bf5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/3a4a95b7da0e41e3b31a98358b050121, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/06e15ecea86343fbab386b1659f9d69c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2918bb26cd104435acdea2a6a24d1e99, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/693c46b1840a4f2893ca4b633a221d7e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/1d8d9b3e6aff4b639e8b6189e4b88703, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/edbe63e5e053402485e9d1db64973f79, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/7f18b6210443438e98ecbffa187564de, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2eda9906e3404aa6a3c3c20769eb0aff, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/6a03e6d6167a4a21a8b4e34d74580ee5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/0951c63fd0744ab8939ecd6480c86c3b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/1a41b1b2842244228f1e8bb68de5e5ba, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/d3fb6e01a51748f5b9b864364f67804f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2fb4bbef2a80435cb31c7e245bc27315] to archive 2024-12-09T01:43:54,424 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T01:43:54,425 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/ca9f66c95fec4d87a8129559a638fbc7 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/ca9f66c95fec4d87a8129559a638fbc7 2024-12-09T01:43:54,426 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/93bfc537b6ed4c019708848a726eb884 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/93bfc537b6ed4c019708848a726eb884 2024-12-09T01:43:54,427 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/f9024eebdc214a9fb7a2c2eb4bffcc68 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/f9024eebdc214a9fb7a2c2eb4bffcc68 2024-12-09T01:43:54,428 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/1f1568359e0f4f83a8c41b1d0997eb99 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/1f1568359e0f4f83a8c41b1d0997eb99 2024-12-09T01:43:54,429 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/8d90d7a849e144c8a0bfb26ffa9d796b to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/8d90d7a849e144c8a0bfb26ffa9d796b 2024-12-09T01:43:54,429 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/92dd1c62bed643cf80d0137dd556d0e9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/92dd1c62bed643cf80d0137dd556d0e9 2024-12-09T01:43:54,430 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2fb6b5e2e77c46859ae65103addd618e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2fb6b5e2e77c46859ae65103addd618e 2024-12-09T01:43:54,431 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/8d7e461a404d4cecbd7dafaaf8064c80 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/8d7e461a404d4cecbd7dafaaf8064c80 2024-12-09T01:43:54,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/9ba75082de6f46a49010b565a00108f7 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/9ba75082de6f46a49010b565a00108f7 2024-12-09T01:43:54,432 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/c7dbcac6f2c24434a185b202767700b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/c7dbcac6f2c24434a185b202767700b9 2024-12-09T01:43:54,433 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/0dc51fe951b04e8fbcabe61133271f35 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/0dc51fe951b04e8fbcabe61133271f35 2024-12-09T01:43:54,434 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/56a881e3565843bc8b85b94335b10b80 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/56a881e3565843bc8b85b94335b10b80 2024-12-09T01:43:54,435 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/e32d8cccd1854c5597a40acf0a6f7bf5 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/e32d8cccd1854c5597a40acf0a6f7bf5 2024-12-09T01:43:54,435 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/3a4a95b7da0e41e3b31a98358b050121 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/3a4a95b7da0e41e3b31a98358b050121 2024-12-09T01:43:54,436 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/06e15ecea86343fbab386b1659f9d69c to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/06e15ecea86343fbab386b1659f9d69c 2024-12-09T01:43:54,437 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2918bb26cd104435acdea2a6a24d1e99 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2918bb26cd104435acdea2a6a24d1e99 2024-12-09T01:43:54,438 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/693c46b1840a4f2893ca4b633a221d7e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/693c46b1840a4f2893ca4b633a221d7e 2024-12-09T01:43:54,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/1d8d9b3e6aff4b639e8b6189e4b88703 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/1d8d9b3e6aff4b639e8b6189e4b88703 2024-12-09T01:43:54,439 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/edbe63e5e053402485e9d1db64973f79 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/edbe63e5e053402485e9d1db64973f79 2024-12-09T01:43:54,440 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/7f18b6210443438e98ecbffa187564de to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/7f18b6210443438e98ecbffa187564de 2024-12-09T01:43:54,441 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2eda9906e3404aa6a3c3c20769eb0aff to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2eda9906e3404aa6a3c3c20769eb0aff 2024-12-09T01:43:54,442 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/6a03e6d6167a4a21a8b4e34d74580ee5 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/6a03e6d6167a4a21a8b4e34d74580ee5 2024-12-09T01:43:54,443 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/0951c63fd0744ab8939ecd6480c86c3b to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/0951c63fd0744ab8939ecd6480c86c3b 2024-12-09T01:43:54,444 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/1a41b1b2842244228f1e8bb68de5e5ba to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/1a41b1b2842244228f1e8bb68de5e5ba 2024-12-09T01:43:54,444 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/d3fb6e01a51748f5b9b864364f67804f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/d3fb6e01a51748f5b9b864364f67804f 2024-12-09T01:43:54,445 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2fb4bbef2a80435cb31c7e245bc27315 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/2fb4bbef2a80435cb31c7e245bc27315 2024-12-09T01:43:54,446 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/5c642edf9e9f44baa1cadf1bbd33d425, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/327059eb6d6b43c5978284b4274632e8, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/e45d81492a0949599caacfff19953b6e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/3d3bc86f39254fa1a70dac7f29714278, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/5971f41b56ec4d27b180644be5c99b99, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/cab8a3d005944004bc451a582519a5bb, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/01d423d3c44545ea8e6eeac41a1dabed, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/3e7a93e86ce44cf595d6a5519779c922, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/ee268eee066540f3b012585022f53942, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/21e57a310f824f17941956766ca96208, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/c196e3bdcd3a4181870eb762ef539c7c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/9b3a8742fc6f4509b76f60c5fc52cd9f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/31d8d9ce9ecc4034b0bdefe51cb8955a, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/6663fa7bd87d48e99a28f9c4b8a0e426, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/11874b8cdf804614a06ded2390f1cd44, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/0c549fa6844144b1a84619c254f9d8af, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/8e4dfc52ab354a118d7981a3fccdedd9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/9d03af4f58d944cc8e062b3e3e414de1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/c983263f477b4a80b8e69990da7783cb, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/409600b38c3e49c18aa4da7dfdf201d5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/e4730e3bdd124813badcb123a05da567, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/9dffe96fd8c44d0b95a81c201b89f34f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/ba32b5fdb5fe4b82a5761070cb25b7cb, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/e92e4175b31f4686abf6559fd0bf470b, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/d80696dcdf0f49deb2fc1d633a2fc74c, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/8867c5db3cec4e68918bcc9b111d0d54] to archive 2024-12-09T01:43:54,447 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T01:43:54,448 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/5c642edf9e9f44baa1cadf1bbd33d425 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/5c642edf9e9f44baa1cadf1bbd33d425 2024-12-09T01:43:54,449 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/327059eb6d6b43c5978284b4274632e8 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/327059eb6d6b43c5978284b4274632e8 2024-12-09T01:43:54,450 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/e45d81492a0949599caacfff19953b6e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/e45d81492a0949599caacfff19953b6e 2024-12-09T01:43:54,451 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/3d3bc86f39254fa1a70dac7f29714278 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/3d3bc86f39254fa1a70dac7f29714278 2024-12-09T01:43:54,451 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/5971f41b56ec4d27b180644be5c99b99 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/5971f41b56ec4d27b180644be5c99b99 2024-12-09T01:43:54,452 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/cab8a3d005944004bc451a582519a5bb to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/cab8a3d005944004bc451a582519a5bb 2024-12-09T01:43:54,453 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/01d423d3c44545ea8e6eeac41a1dabed to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/01d423d3c44545ea8e6eeac41a1dabed 2024-12-09T01:43:54,454 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/3e7a93e86ce44cf595d6a5519779c922 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/3e7a93e86ce44cf595d6a5519779c922 2024-12-09T01:43:54,455 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/ee268eee066540f3b012585022f53942 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/ee268eee066540f3b012585022f53942 2024-12-09T01:43:54,455 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/21e57a310f824f17941956766ca96208 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/21e57a310f824f17941956766ca96208 2024-12-09T01:43:54,456 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/c196e3bdcd3a4181870eb762ef539c7c to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/c196e3bdcd3a4181870eb762ef539c7c 2024-12-09T01:43:54,457 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/9b3a8742fc6f4509b76f60c5fc52cd9f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/9b3a8742fc6f4509b76f60c5fc52cd9f 2024-12-09T01:43:54,458 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/31d8d9ce9ecc4034b0bdefe51cb8955a to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/31d8d9ce9ecc4034b0bdefe51cb8955a 2024-12-09T01:43:54,459 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/6663fa7bd87d48e99a28f9c4b8a0e426 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/6663fa7bd87d48e99a28f9c4b8a0e426 2024-12-09T01:43:54,459 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/11874b8cdf804614a06ded2390f1cd44 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/11874b8cdf804614a06ded2390f1cd44 2024-12-09T01:43:54,460 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/0c549fa6844144b1a84619c254f9d8af to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/0c549fa6844144b1a84619c254f9d8af 2024-12-09T01:43:54,461 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/8e4dfc52ab354a118d7981a3fccdedd9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/8e4dfc52ab354a118d7981a3fccdedd9 2024-12-09T01:43:54,462 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/9d03af4f58d944cc8e062b3e3e414de1 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/9d03af4f58d944cc8e062b3e3e414de1 2024-12-09T01:43:54,462 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/c983263f477b4a80b8e69990da7783cb to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/c983263f477b4a80b8e69990da7783cb 2024-12-09T01:43:54,463 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/409600b38c3e49c18aa4da7dfdf201d5 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/409600b38c3e49c18aa4da7dfdf201d5 2024-12-09T01:43:54,464 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/e4730e3bdd124813badcb123a05da567 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/e4730e3bdd124813badcb123a05da567 2024-12-09T01:43:54,465 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/9dffe96fd8c44d0b95a81c201b89f34f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/9dffe96fd8c44d0b95a81c201b89f34f 2024-12-09T01:43:54,466 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/ba32b5fdb5fe4b82a5761070cb25b7cb to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/ba32b5fdb5fe4b82a5761070cb25b7cb 2024-12-09T01:43:54,466 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/e92e4175b31f4686abf6559fd0bf470b to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/e92e4175b31f4686abf6559fd0bf470b 2024-12-09T01:43:54,467 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/d80696dcdf0f49deb2fc1d633a2fc74c to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/d80696dcdf0f49deb2fc1d633a2fc74c 2024-12-09T01:43:54,468 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/8867c5db3cec4e68918bcc9b111d0d54 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/8867c5db3cec4e68918bcc9b111d0d54 2024-12-09T01:43:54,470 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/a8c30e8d21564c51b22f8d56af71f732, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/94421a0b2a9044488603ce38c98e5fe1, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/d80800658ddd486ba4e52213f1e98f08, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/4a09a250d7a84389974b1b6d74090a1d, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/147bb0c535a3417a814ec8e43848cdb5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/c2b1b091040f40b380741ce10e3d874f, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/15b2f7037b504aa69bfbfd6c9bbf9617, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/f2dda5cc00e849c98bd2f9b49b7873bd, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/65fcfba4b5f74263afa1fc0aee829f1e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/718227b39ad34a0888c95cbfbc878694, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/6b6e0a4f22ee4424acfb3c2de74725be, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/07cd7f8361514cae81b86f10240726b9, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/49ab6a902153446facec31c8ecd79c29, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/bf9c40a57eba4756bc6136a4dd2c1792, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/1f0e0601be6a4d218740549705dc5994, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/f55df748e4f14cc5beaadd668cf55904, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/e70cd97452214059948cdf63b8b9f9e8, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/416b71c4f039476f98cc0493575ac6d5, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/5137b72cb056476f9697558c39453dfc, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/261de500be644ba191e1b81d2bc45b9e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/55600e481ac44e4e901ce7d499ad4d3e, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/65f9c09d665a4952a0d5ecf4ec937a14, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/cdd156ed72234d31866bbb58d1ad13b6, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/26b3cc34adeb48ba937e7ecedfdd4954, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/797d779306b246648892e547cf9549f2, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/08efa270af60458791a9a34b49d45cb6] to archive 2024-12-09T01:43:54,470 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-09T01:43:54,471 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/a8c30e8d21564c51b22f8d56af71f732 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/a8c30e8d21564c51b22f8d56af71f732 2024-12-09T01:43:54,472 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/94421a0b2a9044488603ce38c98e5fe1 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/94421a0b2a9044488603ce38c98e5fe1 2024-12-09T01:43:54,473 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/d80800658ddd486ba4e52213f1e98f08 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/d80800658ddd486ba4e52213f1e98f08 2024-12-09T01:43:54,474 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/4a09a250d7a84389974b1b6d74090a1d to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/4a09a250d7a84389974b1b6d74090a1d 2024-12-09T01:43:54,474 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/147bb0c535a3417a814ec8e43848cdb5 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/147bb0c535a3417a814ec8e43848cdb5 2024-12-09T01:43:54,475 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/c2b1b091040f40b380741ce10e3d874f to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/c2b1b091040f40b380741ce10e3d874f 2024-12-09T01:43:54,476 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/15b2f7037b504aa69bfbfd6c9bbf9617 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/15b2f7037b504aa69bfbfd6c9bbf9617 2024-12-09T01:43:54,477 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/f2dda5cc00e849c98bd2f9b49b7873bd to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/f2dda5cc00e849c98bd2f9b49b7873bd 2024-12-09T01:43:54,478 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/65fcfba4b5f74263afa1fc0aee829f1e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/65fcfba4b5f74263afa1fc0aee829f1e 2024-12-09T01:43:54,478 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/718227b39ad34a0888c95cbfbc878694 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/718227b39ad34a0888c95cbfbc878694 2024-12-09T01:43:54,479 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/6b6e0a4f22ee4424acfb3c2de74725be to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/6b6e0a4f22ee4424acfb3c2de74725be 2024-12-09T01:43:54,480 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/07cd7f8361514cae81b86f10240726b9 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/07cd7f8361514cae81b86f10240726b9 2024-12-09T01:43:54,481 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/49ab6a902153446facec31c8ecd79c29 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/49ab6a902153446facec31c8ecd79c29 2024-12-09T01:43:54,481 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/bf9c40a57eba4756bc6136a4dd2c1792 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/bf9c40a57eba4756bc6136a4dd2c1792 2024-12-09T01:43:54,482 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/1f0e0601be6a4d218740549705dc5994 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/1f0e0601be6a4d218740549705dc5994 2024-12-09T01:43:54,483 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/f55df748e4f14cc5beaadd668cf55904 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/f55df748e4f14cc5beaadd668cf55904 2024-12-09T01:43:54,484 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/e70cd97452214059948cdf63b8b9f9e8 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/e70cd97452214059948cdf63b8b9f9e8 2024-12-09T01:43:54,485 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/416b71c4f039476f98cc0493575ac6d5 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/416b71c4f039476f98cc0493575ac6d5 2024-12-09T01:43:54,486 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/5137b72cb056476f9697558c39453dfc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/5137b72cb056476f9697558c39453dfc 2024-12-09T01:43:54,487 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/261de500be644ba191e1b81d2bc45b9e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/261de500be644ba191e1b81d2bc45b9e 2024-12-09T01:43:54,488 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/55600e481ac44e4e901ce7d499ad4d3e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/55600e481ac44e4e901ce7d499ad4d3e 2024-12-09T01:43:54,489 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/65f9c09d665a4952a0d5ecf4ec937a14 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/65f9c09d665a4952a0d5ecf4ec937a14 2024-12-09T01:43:54,490 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/cdd156ed72234d31866bbb58d1ad13b6 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/cdd156ed72234d31866bbb58d1ad13b6 2024-12-09T01:43:54,490 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/26b3cc34adeb48ba937e7ecedfdd4954 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/26b3cc34adeb48ba937e7ecedfdd4954 2024-12-09T01:43:54,491 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/797d779306b246648892e547cf9549f2 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/797d779306b246648892e547cf9549f2 2024-12-09T01:43:54,492 DEBUG [StoreCloser-TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/08efa270af60458791a9a34b49d45cb6 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/08efa270af60458791a9a34b49d45cb6 2024-12-09T01:43:54,495 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/recovered.edits/373.seqid, newMaxSeqId=373, maxSeqId=4 2024-12-09T01:43:54,496 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. 2024-12-09T01:43:54,496 DEBUG [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1676): Region close journal for 45d234ddf430a4159b8bc5324edd88bc: Waiting for close lock at 1733708633577Running coprocessor pre-close hooks at 1733708633577Disabling compacts and flushes for region at 1733708633577Disabling writes for close at 1733708633577Obtaining lock to block concurrent updates at 1733708633577Preparing flush snapshotting stores in 45d234ddf430a4159b8bc5324edd88bc at 1733708633577Finished memstore snapshotting TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1733708633578 (+1 ms)Flushing stores of TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc. at 1733708633578Flushing 45d234ddf430a4159b8bc5324edd88bc/A: creating writer at 1733708633578Flushing mob file 45d234ddf430a4159b8bc5324edd88bc/A: appending metadata at 1733708633582 (+4 ms)Flushing mob file 45d234ddf430a4159b8bc5324edd88bc/A: closing flushed file at 1733708633582Flushing 45d234ddf430a4159b8bc5324edd88bc/A: appending metadata at 1733708633989 (+407 ms)Flushing 45d234ddf430a4159b8bc5324edd88bc/A: closing flushed file at 1733708633989Flushing 45d234ddf430a4159b8bc5324edd88bc/B: creating writer at 1733708634396 (+407 ms)Flushing 45d234ddf430a4159b8bc5324edd88bc/B: appending metadata at 1733708634398 (+2 ms)Flushing 45d234ddf430a4159b8bc5324edd88bc/B: closing flushed file at 1733708634398Flushing 45d234ddf430a4159b8bc5324edd88bc/C: creating writer at 1733708634404 (+6 ms)Flushing 45d234ddf430a4159b8bc5324edd88bc/C: appending metadata at 1733708634406 (+2 ms)Flushing 45d234ddf430a4159b8bc5324edd88bc/C: closing flushed file at 1733708634407 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77b5b750: reopening flushed file at 1733708634413 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@72e3dce0: reopening flushed file at 1733708634416 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41587a2c: reopening flushed file at 1733708634419 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 45d234ddf430a4159b8bc5324edd88bc in 846ms, sequenceid=370, compaction requested=false at 1733708634423 (+4 ms)Writing region close event to WAL at 1733708634493 (+70 ms)Running coprocessor post-close hooks at 1733708634496 (+3 ms)Closed at 1733708634496 2024-12-09T01:43:54,497 INFO [RS_CLOSE_REGION-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(157): Closed 45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:54,498 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=45d234ddf430a4159b8bc5324edd88bc, regionState=CLOSED 2024-12-09T01:43:54,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure 45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 because future has completed 2024-12-09T01:43:54,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-12-09T01:43:54,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseRegionProcedure 45d234ddf430a4159b8bc5324edd88bc, server=ef6f18c58dc9,38627,1733708453943 in 1.0750 sec 2024-12-09T01:43:54,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=174, resume processing ppid=173 2024-12-09T01:43:54,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, ppid=173, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=45d234ddf430a4159b8bc5324edd88bc, UNASSIGN in 1.0790 sec 2024-12-09T01:43:54,504 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-12-09T01:43:54,504 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.0820 sec 2024-12-09T01:43:54,505 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733708634505"}]},"ts":"1733708634505"} 2024-12-09T01:43:54,506 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-12-09T01:43:54,506 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-12-09T01:43:54,507 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.0900 sec 2024-12-09T01:43:54,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-12-09T01:43:54,553 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:54,554 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-12-09T01:43:54,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:43:54,555 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:43:54,556 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=176, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:43:54,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-12-09T01:43:54,558 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:54,559 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A, FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B, FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C, FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/recovered.edits] 2024-12-09T01:43:54,561 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/1b61be849401472d9a4e90ce41d3dc81 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/1b61be849401472d9a4e90ce41d3dc81 2024-12-09T01:43:54,562 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/665e671c9ef04d319e2fcd2e2b410e95 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/A/665e671c9ef04d319e2fcd2e2b410e95 2024-12-09T01:43:54,563 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/bbe9921e9e504b8392371c0c2f21ddd0 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/bbe9921e9e504b8392371c0c2f21ddd0 2024-12-09T01:43:54,564 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/c6e71284f7e94e97914b0b477c684c35 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/B/c6e71284f7e94e97914b0b477c684c35 2024-12-09T01:43:54,566 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/d3784f921dd2461289fa4cba6136b838 to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/d3784f921dd2461289fa4cba6136b838 2024-12-09T01:43:54,567 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/eb6aba9b79cc4108b38475821e6c541e to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/C/eb6aba9b79cc4108b38475821e6c541e 2024-12-09T01:43:54,569 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/recovered.edits/373.seqid to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc/recovered.edits/373.seqid 2024-12-09T01:43:54,569 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/default/TestAcidGuarantees/45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:54,569 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-12-09T01:43:54,570 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T01:43:54,570 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-12-09T01:43:54,572 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092e1f4839b73746868a7d97c3fbca4520_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092e1f4839b73746868a7d97c3fbca4520_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:54,573 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092f0d0fbebdab443eb83637909a4ec49f_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092f0d0fbebdab443eb83637909a4ec49f_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:54,574 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092fdf8b1b10184d1990f3511a025bb381_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412092fdf8b1b10184d1990f3511a025bb381_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:54,575 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120935a310fd5b2e43bbbfafd20ce557be90_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120935a310fd5b2e43bbbfafd20ce557be90_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:54,575 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120938ec5ba4295a4c7cbf743e4a0874af3b_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120938ec5ba4295a4c7cbf743e4a0874af3b_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:54,576 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209569cd10521104baa9c20bc2fda44e672_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209569cd10521104baa9c20bc2fda44e672_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:54,577 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096514b97ac60e4274b4073ecb92ba4904_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096514b97ac60e4274b4073ecb92ba4904_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:54,578 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096891cfb63c1d4409a78544c3eec481a0_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412096891cfb63c1d4409a78544c3eec481a0_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:54,578 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209831e6b185fb84289ae037cf98a7ed130_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209831e6b185fb84289ae037cf98a7ed130_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:54,579 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120986fe2fbaf87449a781e0d402f79912ed_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024120986fe2fbaf87449a781e0d402f79912ed_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:54,580 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098d93ce47e1a5459285869db68c863ec8_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412098d93ce47e1a5459285869db68c863ec8_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:54,581 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099adc93a925dc43eaaafafa25b78497a4_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202412099adc93a925dc43eaaafafa25b78497a4_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:54,581 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b32fe2339f994a029d7f88527e5f0215_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b32fe2339f994a029d7f88527e5f0215_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:54,582 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b3aaa728ba1e4f77ad125cc0ba6e8f83_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209b3aaa728ba1e4f77ad125cc0ba6e8f83_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:54,583 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209baa30410c76f4ddcab2038ad71aea018_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209baa30410c76f4ddcab2038ad71aea018_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:54,584 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c3d0ade1a77c4578aefef4e8b1ef8f6a_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209c3d0ade1a77c4578aefef4e8b1ef8f6a_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:54,585 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209dcf372c8ed854ccba3f3d1217e639374_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209dcf372c8ed854ccba3f3d1217e639374_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:54,585 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209edc9d0e911ba45a1b876d47d71579cf0_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209edc9d0e911ba45a1b876d47d71579cf0_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:54,586 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f1ee0a1997e44b33be3ae016ae1fabe6_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209f1ee0a1997e44b33be3ae016ae1fabe6_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:54,587 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209fd16d76628c34768860a7640b4043338_45d234ddf430a4159b8bc5324edd88bc to hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241209fd16d76628c34768860a7640b4043338_45d234ddf430a4159b8bc5324edd88bc 2024-12-09T01:43:54,587 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-12-09T01:43:54,589 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=176, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:43:54,591 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-12-09T01:43:54,593 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-12-09T01:43:54,594 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=176, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:43:54,594 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-12-09T01:43:54,594 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1733708634594"}]},"ts":"9223372036854775807"} 2024-12-09T01:43:54,595 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-12-09T01:43:54,595 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 45d234ddf430a4159b8bc5324edd88bc, NAME => 'TestAcidGuarantees,,1733708601932.45d234ddf430a4159b8bc5324edd88bc.', STARTKEY => '', ENDKEY => ''}] 2024-12-09T01:43:54,595 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-12-09T01:43:54,596 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1733708634595"}]},"ts":"9223372036854775807"} 2024-12-09T01:43:54,597 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-12-09T01:43:54,597 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=176, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-12-09T01:43:54,598 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 43 msec 2024-12-09T01:43:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=37995 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-12-09T01:43:54,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-12-09T01:43:54,663 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-12-09T01:43:54,672 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 253), OpenFileDescriptor=449 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=547 (was 673), ProcessCount=11 (was 11), AvailableMemoryMB=11078 (was 11170) 2024-12-09T01:43:54,672 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-09T01:43:54,672 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T01:43:54,672 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T01:43:54,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:54,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:54,673 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-09T01:43:54,673 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T01:43:54,673 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1240002821, stopped=false 2024-12-09T01:43:54,673 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ef6f18c58dc9,37995,1733708453073 2024-12-09T01:43:54,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38627-0x100746f49bc0001, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T01:43:54,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T01:43:54,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38627-0x100746f49bc0001, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:43:54,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:43:54,675 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T01:43:54,675 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-09T01:43:54,676 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T01:43:54,676 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:54,676 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ef6f18c58dc9,38627,1733708453943' ***** 2024-12-09T01:43:54,676 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38627-0x100746f49bc0001, quorum=127.0.0.1:50027, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T01:43:54,676 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T01:43:54,676 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-09T01:43:54,677 INFO [RS:0;ef6f18c58dc9:38627 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T01:43:54,677 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-09T01:43:54,677 INFO [RS:0;ef6f18c58dc9:38627 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T01:43:54,677 INFO [RS:0;ef6f18c58dc9:38627 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T01:43:54,677 INFO [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer(959): stopping server ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:54,677 INFO [RS:0;ef6f18c58dc9:38627 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T01:43:54,677 INFO [RS:0;ef6f18c58dc9:38627 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ef6f18c58dc9:38627. 2024-12-09T01:43:54,677 DEBUG [RS:0;ef6f18c58dc9:38627 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T01:43:54,677 DEBUG [RS:0;ef6f18c58dc9:38627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:54,678 INFO [RS:0;ef6f18c58dc9:38627 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T01:43:54,678 INFO [RS:0;ef6f18c58dc9:38627 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T01:43:54,678 INFO [RS:0;ef6f18c58dc9:38627 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T01:43:54,678 INFO [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-09T01:43:54,678 INFO [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-09T01:43:54,679 DEBUG [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-09T01:43:54,679 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-09T01:43:54,679 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-09T01:43:54,679 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-09T01:43:54,679 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T01:43:54,679 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T01:43:54,679 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-12-09T01:43:54,679 DEBUG [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T01:43:54,703 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740/.tmp/info/8aac78573ce44bb8b14162fb8f3eabc4 is 95, key is TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7./info:/1733708489622/DeleteFamily/seqid=0 2024-12-09T01:43:54,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742544_1720 (size=6514) 2024-12-09T01:43:54,771 INFO [regionserver/ef6f18c58dc9:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T01:43:54,879 DEBUG [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T01:43:55,079 DEBUG [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T01:43:55,107 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740/.tmp/info/8aac78573ce44bb8b14162fb8f3eabc4 2024-12-09T01:43:55,125 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740/.tmp/ns/b6a3494bb78c465c9f197f18ee26f3c4 is 93, key is TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7./ns:/1733708489543/DeleteFamily/seqid=0 2024-12-09T01:43:55,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742545_1721 (size=5906) 2024-12-09T01:43:55,280 DEBUG [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T01:43:55,480 DEBUG [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T01:43:55,528 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740/.tmp/ns/b6a3494bb78c465c9f197f18ee26f3c4 2024-12-09T01:43:55,529 INFO [regionserver/ef6f18c58dc9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-09T01:43:55,529 INFO [regionserver/ef6f18c58dc9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-09T01:43:55,545 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740/.tmp/rep_barrier/bdf1be64fda0480a9db7378838b454d0 is 102, key is TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7./rep_barrier:/1733708489543/DeleteFamily/seqid=0 2024-12-09T01:43:55,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742546_1722 (size=6025) 2024-12-09T01:43:55,680 DEBUG [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T01:43:55,880 DEBUG [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T01:43:55,949 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740/.tmp/rep_barrier/bdf1be64fda0480a9db7378838b454d0 2024-12-09T01:43:55,967 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740/.tmp/table/63d99252844740509c11d14aa7b6f9bb is 96, key is TestAcidGuarantees,,1733708457484.88980a4de80c5830889346818652c3a7./table:/1733708489543/DeleteFamily/seqid=0 2024-12-09T01:43:55,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742547_1723 (size=5936) 2024-12-09T01:43:56,081 DEBUG [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T01:43:56,281 DEBUG [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-09T01:43:56,371 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740/.tmp/table/63d99252844740509c11d14aa7b6f9bb 2024-12-09T01:43:56,374 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740/.tmp/info/8aac78573ce44bb8b14162fb8f3eabc4 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740/info/8aac78573ce44bb8b14162fb8f3eabc4 2024-12-09T01:43:56,377 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740/info/8aac78573ce44bb8b14162fb8f3eabc4, entries=12, sequenceid=90, filesize=6.4 K 2024-12-09T01:43:56,378 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740/.tmp/ns/b6a3494bb78c465c9f197f18ee26f3c4 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740/ns/b6a3494bb78c465c9f197f18ee26f3c4 2024-12-09T01:43:56,380 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740/ns/b6a3494bb78c465c9f197f18ee26f3c4, entries=8, sequenceid=90, filesize=5.8 K 2024-12-09T01:43:56,381 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740/.tmp/rep_barrier/bdf1be64fda0480a9db7378838b454d0 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740/rep_barrier/bdf1be64fda0480a9db7378838b454d0 2024-12-09T01:43:56,383 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740/rep_barrier/bdf1be64fda0480a9db7378838b454d0, entries=6, sequenceid=90, filesize=5.9 K 2024-12-09T01:43:56,384 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740/.tmp/table/63d99252844740509c11d14aa7b6f9bb as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740/table/63d99252844740509c11d14aa7b6f9bb 2024-12-09T01:43:56,387 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740/table/63d99252844740509c11d14aa7b6f9bb, entries=7, sequenceid=90, filesize=5.8 K 2024-12-09T01:43:56,387 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1708ms, sequenceid=90, compaction requested=false 2024-12-09T01:43:56,391 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-12-09T01:43:56,391 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T01:43:56,392 INFO [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-09T01:43:56,392 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733708634678Running coprocessor pre-close hooks at 1733708634679 (+1 ms)Disabling compacts and flushes for region at 1733708634679Disabling writes for close at 1733708634679Obtaining lock to block concurrent updates at 1733708634679Preparing flush snapshotting stores in 1588230740 at 1733708634679Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1733708634679Flushing stores of hbase:meta,,1.1588230740 at 1733708634680 (+1 ms)Flushing 1588230740/info: creating writer at 1733708634680Flushing 1588230740/info: appending metadata at 1733708634702 (+22 ms)Flushing 1588230740/info: closing flushed file at 1733708634702Flushing 1588230740/ns: creating writer at 1733708635110 (+408 ms)Flushing 1588230740/ns: appending metadata at 1733708635125 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733708635125Flushing 1588230740/rep_barrier: creating writer at 1733708635531 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1733708635545 (+14 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1733708635545Flushing 1588230740/table: creating writer at 1733708635952 (+407 ms)Flushing 1588230740/table: appending metadata at 1733708635967 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733708635967Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19a6dfe1: reopening flushed file at 1733708636373 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8eec72b: reopening flushed file at 1733708636377 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@704ba7a5: reopening flushed file at 1733708636381 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34af863f: reopening flushed file at 1733708636384 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1708ms, sequenceid=90, compaction requested=false at 1733708636387 (+3 ms)Writing region close event to WAL at 1733708636389 (+2 ms)Running coprocessor post-close hooks at 1733708636391 (+2 ms)Closed at 1733708636392 (+1 ms) 2024-12-09T01:43:56,392 DEBUG [RS_CLOSE_META-regionserver/ef6f18c58dc9:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T01:43:56,473 INFO [regionserver/ef6f18c58dc9:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-12-09T01:43:56,481 INFO [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer(976): stopping server ef6f18c58dc9,38627,1733708453943; all regions closed. 2024-12-09T01:43:56,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741834_1010 (size=25344) 2024-12-09T01:43:56,489 DEBUG [RS:0;ef6f18c58dc9:38627 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/oldWALs 2024-12-09T01:43:56,489 INFO [RS:0;ef6f18c58dc9:38627 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef6f18c58dc9%2C38627%2C1733708453943.meta:.meta(num 1733708457063) 2024-12-09T01:43:56,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741833_1009 (size=24558651) 2024-12-09T01:43:56,492 DEBUG [RS:0;ef6f18c58dc9:38627 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/oldWALs 2024-12-09T01:43:56,492 INFO [RS:0;ef6f18c58dc9:38627 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ef6f18c58dc9%2C38627%2C1733708453943:(num 1733708456632) 2024-12-09T01:43:56,492 DEBUG [RS:0;ef6f18c58dc9:38627 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T01:43:56,492 INFO [RS:0;ef6f18c58dc9:38627 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T01:43:56,492 INFO [RS:0;ef6f18c58dc9:38627 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T01:43:56,493 INFO [RS:0;ef6f18c58dc9:38627 {}] hbase.ChoreService(370): Chore service for: regionserver/ef6f18c58dc9:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-09T01:43:56,493 INFO [RS:0;ef6f18c58dc9:38627 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T01:43:56,493 INFO [regionserver/ef6f18c58dc9:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T01:43:56,494 INFO [RS:0;ef6f18c58dc9:38627 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38627 2024-12-09T01:43:56,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38627-0x100746f49bc0001, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ef6f18c58dc9,38627,1733708453943 2024-12-09T01:43:56,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T01:43:56,498 INFO [RS:0;ef6f18c58dc9:38627 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T01:43:56,499 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ef6f18c58dc9,38627,1733708453943] 2024-12-09T01:43:56,502 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ef6f18c58dc9,38627,1733708453943 already deleted, retry=false 2024-12-09T01:43:56,502 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ef6f18c58dc9,38627,1733708453943 expired; onlineServers=0 2024-12-09T01:43:56,502 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ef6f18c58dc9,37995,1733708453073' ***** 2024-12-09T01:43:56,502 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T01:43:56,502 INFO [M:0;ef6f18c58dc9:37995 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-09T01:43:56,502 INFO [M:0;ef6f18c58dc9:37995 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-09T01:43:56,502 DEBUG [M:0;ef6f18c58dc9:37995 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T01:43:56,502 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T01:43:56,502 DEBUG [M:0;ef6f18c58dc9:37995 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T01:43:56,502 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster-HFileCleaner.large.0-1733708456326 {}] cleaner.HFileCleaner(306): Exit Thread[master/ef6f18c58dc9:0:becomeActiveMaster-HFileCleaner.large.0-1733708456326,5,FailOnTimeoutGroup] 2024-12-09T01:43:56,502 DEBUG [master/ef6f18c58dc9:0:becomeActiveMaster-HFileCleaner.small.0-1733708456327 {}] cleaner.HFileCleaner(306): Exit Thread[master/ef6f18c58dc9:0:becomeActiveMaster-HFileCleaner.small.0-1733708456327,5,FailOnTimeoutGroup] 2024-12-09T01:43:56,503 INFO [M:0;ef6f18c58dc9:37995 {}] hbase.ChoreService(370): Chore service for: master/ef6f18c58dc9:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-09T01:43:56,503 INFO [M:0;ef6f18c58dc9:37995 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-09T01:43:56,503 DEBUG [M:0;ef6f18c58dc9:37995 {}] master.HMaster(1795): Stopping service threads 2024-12-09T01:43:56,503 INFO [M:0;ef6f18c58dc9:37995 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T01:43:56,503 INFO [M:0;ef6f18c58dc9:37995 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-09T01:43:56,503 ERROR [M:0;ef6f18c58dc9:37995 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-12-09T01:43:56,504 INFO [M:0;ef6f18c58dc9:37995 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T01:43:56,504 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T01:43:56,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T01:43:56,504 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T01:43:56,504 DEBUG [M:0;ef6f18c58dc9:37995 {}] zookeeper.ZKUtil(347): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T01:43:56,505 WARN [M:0;ef6f18c58dc9:37995 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T01:43:56,505 INFO [M:0;ef6f18c58dc9:37995 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/.lastflushedseqids 2024-12-09T01:43:56,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742548_1724 (size=72) 2024-12-09T01:43:56,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38627-0x100746f49bc0001, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T01:43:56,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38627-0x100746f49bc0001, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T01:43:56,602 INFO [RS:0;ef6f18c58dc9:38627 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T01:43:56,602 INFO [RS:0;ef6f18c58dc9:38627 {}] regionserver.HRegionServer(1031): Exiting; stopping=ef6f18c58dc9,38627,1733708453943; zookeeper connection closed. 2024-12-09T01:43:56,602 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@39f13ce8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@39f13ce8 2024-12-09T01:43:56,603 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-09T01:43:56,914 INFO [M:0;ef6f18c58dc9:37995 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-09T01:43:56,915 INFO [M:0;ef6f18c58dc9:37995 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T01:43:56,915 DEBUG [M:0;ef6f18c58dc9:37995 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T01:43:56,915 INFO [M:0;ef6f18c58dc9:37995 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:43:56,915 DEBUG [M:0;ef6f18c58dc9:37995 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:43:56,915 DEBUG [M:0;ef6f18c58dc9:37995 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T01:43:56,915 DEBUG [M:0;ef6f18c58dc9:37995 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:43:56,915 INFO [M:0;ef6f18c58dc9:37995 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=797.50 KB heapSize=978.41 KB 2024-12-09T01:43:56,931 DEBUG [M:0;ef6f18c58dc9:37995 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7781855f15e943f0a8393a456df99ac9 is 82, key is hbase:meta,,1/info:regioninfo/1733708457163/Put/seqid=0 2024-12-09T01:43:56,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742549_1725 (size=5672) 2024-12-09T01:43:57,334 INFO [M:0;ef6f18c58dc9:37995 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2235 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7781855f15e943f0a8393a456df99ac9 2024-12-09T01:43:57,354 DEBUG [M:0;ef6f18c58dc9:37995 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3bde05c4b1ec4910a2051a2e91c4e90d is 2479, key is \x00\x00\x00\x00\x00\x00\x00\x93/proc:d/1733708605000/Put/seqid=0 2024-12-09T01:43:57,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742550_1726 (size=46401) 2024-12-09T01:43:57,758 INFO [M:0;ef6f18c58dc9:37995 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=796.95 KB at sequenceid=2235 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3bde05c4b1ec4910a2051a2e91c4e90d 2024-12-09T01:43:57,761 INFO [M:0;ef6f18c58dc9:37995 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3bde05c4b1ec4910a2051a2e91c4e90d 2024-12-09T01:43:57,776 DEBUG [M:0;ef6f18c58dc9:37995 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d100455870e043a0b5cfdbfec52c53bc is 69, key is ef6f18c58dc9,38627,1733708453943/rs:state/1733708456386/Put/seqid=0 2024-12-09T01:43:57,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073742551_1727 (size=5156) 2024-12-09T01:43:57,781 INFO [M:0;ef6f18c58dc9:37995 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2235 (bloomFilter=true), to=hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d100455870e043a0b5cfdbfec52c53bc 2024-12-09T01:43:57,785 DEBUG [M:0;ef6f18c58dc9:37995 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/7781855f15e943f0a8393a456df99ac9 as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7781855f15e943f0a8393a456df99ac9 2024-12-09T01:43:57,789 INFO [M:0;ef6f18c58dc9:37995 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/7781855f15e943f0a8393a456df99ac9, entries=8, sequenceid=2235, filesize=5.5 K 2024-12-09T01:43:57,790 DEBUG [M:0;ef6f18c58dc9:37995 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3bde05c4b1ec4910a2051a2e91c4e90d as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3bde05c4b1ec4910a2051a2e91c4e90d 2024-12-09T01:43:57,794 INFO [M:0;ef6f18c58dc9:37995 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3bde05c4b1ec4910a2051a2e91c4e90d 2024-12-09T01:43:57,794 INFO [M:0;ef6f18c58dc9:37995 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3bde05c4b1ec4910a2051a2e91c4e90d, entries=176, sequenceid=2235, filesize=45.3 K 2024-12-09T01:43:57,795 DEBUG [M:0;ef6f18c58dc9:37995 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/d100455870e043a0b5cfdbfec52c53bc as hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d100455870e043a0b5cfdbfec52c53bc 2024-12-09T01:43:57,798 INFO [M:0;ef6f18c58dc9:37995 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40841/user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/d100455870e043a0b5cfdbfec52c53bc, entries=1, sequenceid=2235, filesize=5.0 K 2024-12-09T01:43:57,798 INFO [M:0;ef6f18c58dc9:37995 {}] regionserver.HRegion(3140): Finished flush of dataSize ~797.50 KB/816644, heapSize ~978.11 KB/1001584, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 883ms, sequenceid=2235, compaction requested=false 2024-12-09T01:43:57,799 INFO [M:0;ef6f18c58dc9:37995 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T01:43:57,800 DEBUG [M:0;ef6f18c58dc9:37995 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733708636915Disabling compacts and flushes for region at 1733708636915Disabling writes for close at 1733708636915Obtaining lock to block concurrent updates at 1733708636915Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733708636915Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=816644, getHeapSize=1001824, getOffHeapSize=0, getCellsCount=2436 at 1733708636915Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733708636916 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733708636916Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733708636931 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733708636931Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733708637338 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733708637354 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733708637354Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733708637762 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733708637776 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733708637776Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ec267: reopening flushed file at 1733708637785 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16509244: reopening flushed file at 1733708637789 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f300ec5: reopening flushed file at 1733708637794 (+5 ms)Finished flush of dataSize ~797.50 KB/816644, heapSize ~978.11 KB/1001584, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 883ms, sequenceid=2235, compaction requested=false at 1733708637798 (+4 ms)Writing region close event to WAL at 1733708637799 (+1 ms)Closed at 1733708637799 2024-12-09T01:43:57,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33243 is added to blk_1073741830_1006 (size=963142) 2024-12-09T01:43:57,802 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/b4aa9aef-f735-e5cf-bcba-281246d374fb/MasterData/WALs/ef6f18c58dc9,37995,1733708453073/ef6f18c58dc9%2C37995%2C1733708453073.1733708455758 not finished, retry = 0 2024-12-09T01:43:57,903 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-09T01:43:57,903 INFO [M:0;ef6f18c58dc9:37995 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-09T01:43:57,903 INFO [M:0;ef6f18c58dc9:37995 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37995 2024-12-09T01:43:57,904 INFO [M:0;ef6f18c58dc9:37995 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-09T01:43:58,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T01:43:58,006 INFO [M:0;ef6f18c58dc9:37995 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-09T01:43:58,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37995-0x100746f49bc0000, quorum=127.0.0.1:50027, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T01:43:58,011 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@51084f0f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T01:43:58,014 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@370621d9{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T01:43:58,014 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T01:43:58,015 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4828e07b{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T01:43:58,015 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6ff64ba0{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/hadoop.log.dir/,STOPPED} 2024-12-09T01:43:58,019 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T01:43:58,019 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T01:43:58,019 WARN [BP-1038248241-172.17.0.2-1733708449557 heartbeating to localhost/127.0.0.1:40841 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T01:43:58,019 WARN [BP-1038248241-172.17.0.2-1733708449557 heartbeating to localhost/127.0.0.1:40841 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1038248241-172.17.0.2-1733708449557 (Datanode Uuid 36b2a1a8-de5a-4f07-95b2-5596a0203735) service to localhost/127.0.0.1:40841 2024-12-09T01:43:58,022 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/cluster_fb15f3cb-c4dd-1da4-630e-097361e28d71/data/data1/current/BP-1038248241-172.17.0.2-1733708449557 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T01:43:58,022 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/cluster_fb15f3cb-c4dd-1da4-630e-097361e28d71/data/data2/current/BP-1038248241-172.17.0.2-1733708449557 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T01:43:58,023 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T01:43:58,035 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2341a969{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T01:43:58,036 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T01:43:58,036 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T01:43:58,037 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T01:43:58,037 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/e5c3da7e-6fd8-badd-0ec6-274d8cc37029/hadoop.log.dir/,STOPPED} 2024-12-09T01:43:58,054 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-09T01:43:58,189 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down